@causa/workspace-google 1.3.1 → 1.4.0-beta.2

package/dist/graph/environment/fetchers/console-links.js

@@ -0,0 +1,132 @@
+import { googleResources } from '../resources.js';
+/**
+ * The base URL of the Google Cloud console.
+ */
+const CONSOLE_URL = 'https://console.cloud.google.com';
+/**
+ * Builds the URL of a page of the Google Cloud console.
+ *
+ * @param segments The segments of the path of the page, which are encoded.
+ * @param project The GCP project.
+ * @param query Additional query parameters.
+ * @returns The URL.
+ */
+export function consoleUrl(segments, project, query = {}) {
+    const path = segments.map(encodeURIComponent).join('/');
+    const parameters = new URLSearchParams({ project, ...query });
+    return `${CONSOLE_URL}/${path}?${parameters}`;
+}
+/**
+ * The builders of links, keyed by resource type.
+ */
+const LINK_BUILDERS = {
+    'run.googleapis.com/Service': ({ projects, locations, services }) => [
+        {
+            label: 'Cloud Run service',
+            url: consoleUrl(['run', 'detail', locations, services, 'metrics'], projects),
+        },
+        {
+            label: 'Cloud Run service logs',
+            url: consoleUrl(['run', 'detail', locations, services, 'logs'], projects),
+        },
+    ],
+    'pubsub.googleapis.com/Topic': ({ projects, topics }) => [
+        {
+            label: 'Pub/Sub topic',
+            url: consoleUrl(['cloudpubsub', 'topic', 'detail', topics], projects),
+        },
+    ],
+    'pubsub.googleapis.com/Subscription': ({ projects, subscriptions }) => [
+        {
+            label: 'Pub/Sub subscription',
+            url: consoleUrl(['cloudpubsub', 'subscription', 'detail', subscriptions], projects),
+        },
+    ],
+    'cloudtasks.googleapis.com/Queue': ({ projects, locations, queues }) => [
+        {
+            label: 'Cloud Tasks queue',
+            url: consoleUrl(['cloudtasks', 'queue', locations, queues, 'tasks'], projects),
+        },
+    ],
+    'cloudscheduler.googleapis.com/Job': ({ projects }) => [
+        {
+            label: 'Cloud Scheduler jobs',
+            url: consoleUrl(['cloudscheduler'], projects),
+        },
+    ],
+    'spanner.googleapis.com/Instance': ({ projects, instances }) => [
+        {
+            label: 'Spanner instance',
+            url: consoleUrl(['spanner', 'instances', instances, 'details', 'databases'], projects),
+        },
+    ],
+    'spanner.googleapis.com/Database': ({ projects, instances, databases }) => [
+        {
+            label: 'Spanner database',
+            url: consoleUrl([
+                'spanner',
+                'instances',
+                instances,
+                'databases',
+                databases,
+                'details',
+                'tables',
+            ], projects),
+        },
+    ],
+    'firestore.googleapis.com/Database': ({ projects, databases }) => [
+        {
+            label: 'Firestore database',
+            url: consoleUrl([
+                'firestore',
+                'databases',
+                databases === '(default)' ? '-default-' : databases,
+                'data',
+            ], projects),
+        },
+    ],
+    'bigquery.googleapis.com/Dataset': ({ projects, datasets }) => [
+        {
+            label: 'BigQuery dataset',
+            url: consoleUrl(['bigquery'], projects, {
+                p: projects,
+                d: datasets,
+                page: 'dataset',
+            }),
+        },
+    ],
+    'bigquery.googleapis.com/Table': ({ projects, datasets, tables }) => [
+        {
+            label: 'BigQuery table',
+            url: consoleUrl(['bigquery'], projects, {
+                p: projects,
+                d: datasets,
+                t: tables,
+                page: 'table',
+            }),
+        },
+    ],
+    'compute.googleapis.com/UrlMap': ({ projects }) => [
+        {
+            label: 'Load balancers',
+            url: consoleUrl(['net-services', 'loadbalancing', 'list', 'loadBalancers'], projects),
+        },
+    ],
+};
+/**
+ * Builds links to the Google Cloud console for the nodes' resources.
+ */
+export const googleConsoleLinks = {
+    name: 'googleConsoleLinks',
+    description: 'The `links` of nodes whose resource is a Google Cloud resource, to the Google Cloud console.',
+    fetch: async (environment) => {
+        const nodes = {};
+        for (const { node, type, segments } of googleResources(environment)) {
+            const builder = LINK_BUILDERS[type];
+            if (builder && segments.projects) {
+                nodes[node.id] = { data: { links: builder(segments) } };
+            }
+        }
+        return { nodes };
+    },
+};

package/dist/graph/environment/fetchers/error-reporting.d.ts

@@ -0,0 +1,5 @@
+import { type GraphEnvironmentFetcher } from '@causa/workspace-core';
+/**
+ * Reads the open groups of errors of Cloud Run services from Error Reporting, as alerts.
+ */
+export declare const googleErrorReporting: GraphEnvironmentFetcher;

package/dist/graph/environment/fetchers/error-reporting.js

@@ -0,0 +1,98 @@
+import { GraphAlertSeverity, } from '@causa/workspace-core';
+import { googleResources } from '../resources.js';
+import { consoleUrl } from './console-links.js';
+/**
+ * The longest period over which Error Reporting can list groups. Periods always end now.
+ */
+const PERIOD = 'PERIOD_30_DAYS';
+/**
+ * The length of {@link PERIOD}, in milliseconds.
+ */
+const PERIOD_LENGTH = 30 * 24 * 3600 * 1000;
+/**
+ * Converts the statistics of an open error group to an alert, titled with the first line of the message of the group's
+ * representative error.
+ *
+ * @param stats The statistics of the group.
+ * @param project The GCP project of the group.
+ * @returns The alert, or `undefined` if the group has never been seen.
+ */
+function toGraphAlert(stats, project) {
+    if (!stats.firstSeenTime) {
+        return undefined;
+    }
+    const groupId = stats.group?.groupId ?? undefined;
+    const message = stats.representative?.message?.split('\n', 1)[0].trim();
+    return {
+        title: message || `Error group ${groupId}`,
+        severity: GraphAlertSeverity.Error,
+        openedAt: new Date(stats.firstSeenTime),
+        ...(groupId
+            ? { url: consoleUrl(['errors', 'detail', groupId], project) }
+            : {}),
+    };
+}
+/**
+ * Reads the open groups of errors of Cloud Run services from Error Reporting, as alerts.
+ */
+export const googleErrorReporting = {
+    name: 'googleErrorReporting',
+    description: 'The `alerts` of Cloud Run services for their open Error Reporting groups, i.e. groups with occurrences during the last 30 days, first seen before the end of the evaluation window. Acknowledged, resolved, and muted groups are excluded.',
+    fetch: async (environment) => {
+        const services = googleResources(environment, 'run.googleapis.com/Service', 'service');
+        if (services.length === 0) {
+            return {};
+        }
+        const { at } = environment;
+        if (Date.now() - at.getTime() > PERIOD_LENGTH) {
+            return {
+                warnings: [
+                    {
+                        message: 'Error groups cannot be listed for an evaluation window ending more than 30 days ago.',
+                    },
+                ],
+            };
+        }
+        const { GoogleApisService } = await import('../../../services/google-apis.js');
+        const client = await environment.context
+            .service(GoogleApisService)
+            .getClient('clouderrorreporting', 'v1beta1', {});
+        const nodes = {};
+        const warnings = [];
+        await Promise.all(services.map(async ({ node, segments }) => {
+            const stats = [];
+            try {
+                let pageToken;
+                do {
+                    const { data } = await client.projects.groupStats.list({
+                        projectName: `projects/${segments.projects}`,
+                        'serviceFilter.service': segments.services,
+                        'timeRange.period': PERIOD,
+                        pageSize: 100,
+                        pageToken,
+                    });
+                    stats.push(...(data.errorGroupStats ?? []));
+                    pageToken = data.nextPageToken ?? undefined;
+                } while (pageToken);
+            }
+            catch (error) {
+                warnings.push({
+                    message: `Failed to list the error groups of '${node.id}': ${error.message ?? error}`,
+                });
+                return;
+            }
+            const alerts = stats.flatMap((groupStats) => {
+                // An unspecified resolution status is interpreted as open.
+                const status = groupStats.group?.resolutionStatus ?? 'OPEN';
+                const alert = status === 'OPEN'
+                    ? toGraphAlert(groupStats, segments.projects)
+                    : undefined;
+                return alert && alert.openedAt < at ? [alert] : [];
+            });
+            if (alerts.length > 0) {
+                nodes[node.id] = { alerts };
+            }
+        }));
+        return { nodes, warnings };
+    },
+};

package/dist/graph/environment/fetchers/index.d.ts

@@ -0,0 +1,5 @@
+import type { GraphEnvironmentFetcher } from '@causa/workspace-core';
+/**
+ * The fetchers of environment data and metrics for Google Cloud.
+ */
+export declare const GOOGLE_GRAPH_ENVIRONMENT_FETCHERS: GraphEnvironmentFetcher[];

package/dist/graph/environment/fetchers/index.js

@@ -0,0 +1,17 @@
+import { googleCloudMonitoringAlerts } from './cloud-monitoring-alerts.js';
+import { googleCloudMonitoring } from './cloud-monitoring.js';
+import { googleCloudRun } from './cloud-run.js';
+import { googleCloudScheduler } from './cloud-scheduler.js';
+import { googleConsoleLinks } from './console-links.js';
+import { googleErrorReporting } from './error-reporting.js';
+/**
+ * The fetchers of environment data and metrics for Google Cloud.
+ */
+export const GOOGLE_GRAPH_ENVIRONMENT_FETCHERS = [
+    googleCloudMonitoring,
+    googleCloudMonitoringAlerts,
+    googleCloudRun,
+    googleCloudScheduler,
+    googleConsoleLinks,
+    googleErrorReporting,
+];

package/dist/graph/environment/index.d.ts

@@ -0,0 +1,9 @@
+export * from './fetchers/index.js';
+export * from './load-balancer-backends.js';
+export * from './monitoring-alerts.js';
+export * from './monitoring-catalog.js';
+export * from './monitoring-mappings.js';
+export * from './monitoring-query.js';
+export * from './monitoring-values.js';
+export * from './resource-resolvers.js';
+export * from './resources.js';

package/dist/graph/environment/index.js

@@ -0,0 +1,9 @@
+export * from './fetchers/index.js';
+export * from './load-balancer-backends.js';
+export * from './monitoring-alerts.js';
+export * from './monitoring-catalog.js';
+export * from './monitoring-mappings.js';
+export * from './monitoring-query.js';
+export * from './monitoring-values.js';
+export * from './resource-resolvers.js';
+export * from './resources.js';

package/dist/graph/environment/load-balancer-backends.d.ts

@@ -0,0 +1,82 @@
+import type { WorkspaceContext } from '@causa/workspace';
+import { GraphEnvironmentFact, type GraphEnvironmentContext, type GraphFactOutput } from '@causa/workspace-core/graph';
+import { type MonitoringLabel, type MonitoringResourceMapping } from './monitoring-mappings.js';
+import { type GoogleResource } from './resources.js';
+/**
+ * Maps load balancer labels to the URL maps of the graph, i.e. API routers.
+ */
+export declare const URL_MAP_MAPPING: MonitoringResourceMapping;
+/**
+ * A backend of load balancers, which routes requests to a resource.
+ */
+export type LoadBalancerBackend = {
+    /**
+     * The kind of the backend.
+     */
+    readonly kind: LoadBalancerBackendKind;
+    /**
+     * The GCP project of the backend.
+     */
+    readonly project: string;
+    /**
+     * The values of the key labels of the kind identifying the backend.
+     */
+    readonly labels: Readonly<Record<MonitoringLabel, string>>;
+    /**
+     * The full resource name of the resource the backend routes requests to, e.g. a Cloud Run service.
+     */
+    readonly resource: string;
+};
+/**
+ * A kind of load balancer backend, routing requests to the resources of nodes of a type.
+ */
+export type LoadBalancerBackendKind = {
+    /**
+     * The type of the nodes requests are routed to.
+     */
+    readonly nodeType: string;
+    /**
+     * The type of the nodes' resource, in their `data.resource.type`.
+     */
+    readonly resourceType: string;
+    /**
+     * The labels of load balancer time series and alerts identifying a backend of this kind within a project.
+     */
+    readonly keyLabels: readonly MonitoringLabel[];
+    /**
+     * Lists the backends of this kind. It throws if some backends cannot be listed.
+     *
+     * @param context The context, from which the `GoogleApisService` is obtained.
+     * @param projects The projects of the graph's URL maps, in which backends are listed.
+     * @param resources The resources of the kind's types, e.g. to only list regional backends in their regions.
+     * @returns The backends.
+     */
+    list(context: WorkspaceContext, projects: readonly string[], resources: readonly GoogleResource[]): Promise<LoadBalancerBackend[]>;
+};
+/**
+ * Serverless network endpoint groups, routing requests to Cloud Run services.
+ * They are listed in the regions of the services, in the projects of the URL maps.
+ */
+export declare const SERVERLESS_NEG_BACKENDS: LoadBalancerBackendKind;
+/**
+ * The kinds of backends through which requests can be routed to nodes.
+ */
+export declare const LOAD_BALANCER_BACKEND_KINDS: readonly LoadBalancerBackendKind[];
+/**
+ * Lists the backends of all kinds, in the projects of the URL maps of the graph. No API is called if the graph has no
+ * URL map. The fact fails if some backends cannot be listed.
+ */
+export declare class LoadBalancerBackendsFact extends GraphEnvironmentFact<readonly LoadBalancerBackend[]> {
+    compute(environment: GraphEnvironmentContext): Promise<GraphFactOutput<readonly LoadBalancerBackend[]>>;
+}
+/**
+ * Builds the mapping of the backends of a kind, from load balancer labels to the resources they route requests to.
+ *
+ * @param kind The kind of backends.
+ * @returns The mapping.
+ */
+export declare function loadBalancerBackendMapping(kind: LoadBalancerBackendKind): MonitoringResourceMapping;
+/**
+ * The mappings of load balancer labels to the resources backends route requests to, one per kind of backend.
+ */
+export declare const LOAD_BALANCER_BACKEND_MAPPINGS: readonly MonitoringResourceMapping[];

package/dist/graph/environment/load-balancer-backends.js

@@ -0,0 +1,126 @@
+import { GraphEnvironmentFact, } from '@causa/workspace-core/graph';
+import { templateMapping, } from './monitoring-mappings.js';
+import { googleResources } from './resources.js';
+/**
+ * Maps load balancer labels to the URL maps of the graph, i.e. API routers.
+ */
+export const URL_MAP_MAPPING = templateMapping({
+    nodeType: 'apiRouter',
+    resourceType: 'compute.googleapis.com/UrlMap',
+    monitoredResource: 'https_lb_rule',
+    name: 'projects/{project}/global/urlMaps/{resource.label.url_map_name}',
+});
+/**
+ * Serverless network endpoint groups, routing requests to Cloud Run services.
+ * They are listed in the regions of the services, in the projects of the URL maps.
+ */
+export const SERVERLESS_NEG_BACKENDS = {
+    nodeType: 'service',
+    resourceType: 'run.googleapis.com/Service',
+    keyLabels: ['resource.label.backend_scope', 'resource.label.backend_name'],
+    list: async (context, projects, resources) => {
+        const locations = new Map();
+        for (const { segments } of resources) {
+            const { projects: project, locations: region } = segments;
+            if (project && region && projects.includes(project)) {
+                locations.set(`projects/${project}/regions/${region}`, {
+                    project,
+                    region,
+                });
+            }
+        }
+        if (locations.size === 0) {
+            return [];
+        }
+        const { GoogleApisService } = await import('../../services/google-apis.js');
+        const client = await context
+            .service(GoogleApisService)
+            .getClient('compute', 'v1', {});
+        const backends = await Promise.all([...locations.values()].map(async ({ project, region }) => {
+            const groups = [];
+            try {
+                let pageToken;
+                do {
+                    const { data } = await client.regionNetworkEndpointGroups.list({
+                        project,
+                        region,
+                        pageToken,
+                    });
+                    groups.push(...(data.items ?? []));
+                    pageToken = data.nextPageToken ?? undefined;
+                } while (pageToken);
+            }
+            catch (error) {
+                throw new Error(`Failed to list the network endpoint groups in project '${project}' and region '${region}': ${error.message ?? error}`, { cause: error });
+            }
+            return groups.flatMap((group) => {
+                const service = group.cloudRun?.service;
+                if (group.networkEndpointType !== 'SERVERLESS' ||
+                    !group.name ||
+                    !service) {
+                    return [];
+                }
+                return [
+                    {
+                        kind: SERVERLESS_NEG_BACKENDS,
+                        project,
+                        labels: {
+                            'resource.label.backend_scope': region,
+                            'resource.label.backend_name': group.name,
+                        },
+                        resource: `projects/${project}/locations/${region}/services/${service}`,
+                    },
+                ];
+            });
+        }));
+        return backends.flat();
+    },
+};
+/**
+ * The kinds of backends through which requests can be routed to nodes.
+ */
+export const LOAD_BALANCER_BACKEND_KINDS = [
+    SERVERLESS_NEG_BACKENDS,
+];
+/**
+ * Lists the backends of all kinds, in the projects of the URL maps of the graph. No API is called if the graph has no
+ * URL map. The fact fails if some backends cannot be listed.
+ */
+export class LoadBalancerBackendsFact extends GraphEnvironmentFact {
+    async compute(environment) {
+        const projects = [
+            ...new Set(googleResources(environment, URL_MAP_MAPPING.resourceType, URL_MAP_MAPPING.nodeType).map(({ segments }) => segments.projects)),
+        ];
+        if (projects.length === 0) {
+            return { value: [] };
+        }
+        const backends = await Promise.all(LOAD_BALANCER_BACKEND_KINDS.map((kind) => kind.list(environment.context, projects, googleResources(environment, kind.resourceType, kind.nodeType))));
+        return { value: backends.flat() };
+    }
+}
+/**
+ * Builds the mapping of the backends of a kind, from load balancer labels to the resources they route requests to.
+ *
+ * @param kind The kind of backends.
+ * @returns The mapping.
+ */
+export function loadBalancerBackendMapping(kind) {
+    return {
+        nodeType: kind.nodeType,
+        resourceType: kind.resourceType,
+        monitoredResource: URL_MAP_MAPPING.monitoredResource,
+        keyLabels: kind.keyLabels,
+        resolver: async (environment) => {
+            const backends = await environment.get(LoadBalancerBackendsFact);
+            const key = (project, labels) => JSON.stringify([project, ...kind.keyLabels.map((l) => labels[l])]);
+            const resources = new Map(backends
+                .filter((b) => b.kind === kind)
+                .map((b) => [key(b.project, b.labels), b.resource]));
+            return ({ project, labels }) => resources.get(key(project, labels));
+        },
+    };
+}
+/**
+ * The mappings of load balancer labels to the resources backends route requests to, one per kind of backend.
+ */
+export const LOAD_BALANCER_BACKEND_MAPPINGS = LOAD_BALANCER_BACKEND_KINDS.map(loadBalancerBackendMapping);

package/dist/graph/environment/monitoring-alerts.d.ts

@@ -0,0 +1,60 @@
+import type { WorkspaceContext } from '@causa/workspace';
+import type { monitoring_v3 } from 'googleapis';
+import { type MonitoringLabel } from './monitoring-mappings.js';
+/**
+ * An alert, as returned by the Cloud Monitoring Alerts API.
+ */
+export type MonitoringAlert = monitoring_v3.Schema$Alert;
+/**
+ * The monitored resource of an alert.
+ */
+export type MonitoringAlertResource = {
+    /**
+     * The monitored resource type, e.g. `cloud_run_revision`.
+     */
+    readonly type: string;
+    /**
+     * The GCP project of the resource: the `project_id` label, or the project from which the alert was listed.
+     */
+    readonly project: string;
+    /**
+     * The labels of the alert, keyed by name. Resource labels take precedence over metric labels with the same name, as
+     * alerts aggregating away the resource type only keep metric labels.
+     */
+    readonly labels: Readonly<Record<string, string>>;
+};
+/**
+ * An alert open at a given time, along with its monitored resource.
+ */
+export type OpenMonitoringAlert = {
+    /**
+     * The alert, as returned by the Alerts API.
+     */
+    readonly alert: MonitoringAlert;
+    /**
+     * The GCP project from which the alert was listed.
+     */
+    readonly project: string;
+    /**
+     * The monitored resource of the alert. It is `undefined` when its type is unknown.
+     */
+    readonly resource?: MonitoringAlertResource;
+};
+/**
+ * Returns the value of a label of the resource of an alert, read by its name regardless of its source, as alerts
+ * aggregating away the resource type only keep metric labels.
+ *
+ * @param resource The resource of the alert.
+ * @param label The label.
+ * @returns The value of the label.
+ */
+export declare function alertLabelValue(resource: MonitoringAlertResource, label: MonitoringLabel): string | undefined;
+/**
+ * Lists the alerts of a project that were open at a given time, along with their monitored resources.
+ *
+ * @param context The context, from which the `GoogleApisService` is obtained.
+ * @param project The GCP project.
+ * @param at The time at which the alerts were open.
+ * @returns The alerts.
+ */
+export declare function listOpenAlerts(context: WorkspaceContext, project: string, at: Date): Promise<OpenMonitoringAlert[]>;

package/dist/graph/environment/monitoring-alerts.js

@@ -0,0 +1,67 @@
+import { monitoringLabelName, } from './monitoring-mappings.js';
+/**
+ * The resource type of alerts whose condition aggregates away the monitored resource type, e.g. a PromQL query
+ * aggregating a metric written against several resource types.
+ */
+const PROMETHEUS_TARGET = 'prometheus_target';
+/**
+ * Returns the value of a label of the resource of an alert, read by its name regardless of its source, as alerts
+ * aggregating away the resource type only keep metric labels.
+ *
+ * @param resource The resource of the alert.
+ * @param label The label.
+ * @returns The value of the label.
+ */
+export function alertLabelValue(resource, label) {
+    return resource.labels[monitoringLabelName(label)];
+}
+/**
+ * Lists the alerts of a project that were open at a given time, along with their monitored resources.
+ *
+ * @param context The context, from which the `GoogleApisService` is obtained.
+ * @param project The GCP project.
+ * @param at The time at which the alerts were open.
+ * @returns The alerts.
+ */
+export async function listOpenAlerts(context, project, at) {
+    const { GoogleApisService } = await import('../../services/google-apis.js');
+    const client = await context
+        .service(GoogleApisService)
+        .getClient('monitoring', 'v3', {});
+    // The API only supports `>=` and `<=` comparisons, and timestamps without fractional seconds. Alerts that are still
+    // open have no closing time.
+    const time = at.toISOString().replace(/\.\d+Z$/, 'Z');
+    const filter = `open_time <= "${time}" AND (state = "OPEN" OR close_time >= "${time}")`;
+    const alerts = [];
+    let pageToken;
+    do {
+        const { data } = await client.projects.alerts.list({
+            parent: `projects/${project}`,
+            filter,
+            pageSize: 1000,
+            pageToken,
+        });
+        alerts.push(...(data.alerts ?? []));
+        pageToken = data.nextPageToken ?? undefined;
+    } while (pageToken);
+    return alerts.map((alert) => {
+        const labels = {
+            ...alert.metric?.labels,
+            ...alert.resource?.labels,
+        };
+        const resourceType = alert.resource?.type;
+        // PromQL alerts have their resource type set to `prometheus_target`. In that case, the actual resource type is read
+        // from the `monitored_resource` label, which the alert is expected to group by.
+        const type = resourceType && resourceType !== PROMETHEUS_TARGET
+            ? resourceType
+            : labels.monitored_resource;
+        if (!type) {
+            return { alert, project };
+        }
+        return {
+            alert,
+            project,
+            resource: { type, project: labels.project_id ?? project, labels },
+        };
+    });
+}