@causa/workspace-google 1.4.0-beta.1 → 1.4.0-beta.2

package/dist/graph/environment/monitoring-catalog.js

@@ -0,0 +1,356 @@
+import { GraphMetricKind, } from '@causa/workspace-core';
+import { CORE_GRAPH_METRIC_DEFINITIONS, } from '@causa/workspace-core/graph';
+import { LOAD_BALANCER_BACKEND_MAPPINGS, URL_MAP_MAPPING, } from './load-balancer-backends.js';
+import { templateMapping, } from './monitoring-mappings.js';
+export const CLOUD_RUN_SERVICE_MAPPING = templateMapping({
+    nodeType: 'service',
+    resourceType: 'run.googleapis.com/Service',
+    monitoredResource: 'cloud_run_revision',
+    name: 'projects/{project}/locations/{resource.label.location}/services/{resource.label.service_name}',
+});
+export const PUBSUB_TOPIC_MAPPING = templateMapping({
+    nodeType: 'brokerTopic',
+    resourceType: 'pubsub.googleapis.com/Topic',
+    monitoredResource: 'pubsub_topic',
+    name: 'projects/{project}/topics/{resource.label.topic_id}',
+});
+export const PUBSUB_SUBSCRIPTION_MAPPING = templateMapping({
+    nodeType: 'brokerSubscription',
+    resourceType: 'pubsub.googleapis.com/Subscription',
+    monitoredResource: 'pubsub_subscription',
+    name: 'projects/{project}/subscriptions/{resource.label.subscription_id}',
+});
+export const CLOUD_TASKS_QUEUE_MAPPING = templateMapping({
+    nodeType: 'queue',
+    resourceType: 'cloudtasks.googleapis.com/Queue',
+    monitoredResource: 'cloud_tasks_queue',
+    name: 'projects/{project}/locations/{resource.label.location}/queues/{resource.label.queue_id}',
+});
+export const SPANNER_INSTANCE_MAPPING = templateMapping({
+    nodeType: 'google.spanner.instance',
+    resourceType: 'spanner.googleapis.com/Instance',
+    monitoredResource: 'spanner_instance',
+    name: 'projects/{project}/instances/{resource.label.instance_id}',
+});
+export const SPANNER_DATABASE_MAPPING = templateMapping({
+    nodeType: 'database',
+    resourceType: 'spanner.googleapis.com/Database',
+    monitoredResource: 'spanner_instance',
+    name: 'projects/{project}/instances/{resource.label.instance_id}/databases/{metric.label.database}',
+});
+export const FIRESTORE_DATABASE_MAPPING = templateMapping({
+    nodeType: 'database',
+    resourceType: 'firestore.googleapis.com/Database',
+    monitoredResource: 'firestore.googleapis.com/Database',
+    name: 'projects/{project}/databases/{resource.label.database_id}',
+});
+export const BIGQUERY_DATASET_MAPPING = templateMapping({
+    nodeType: 'database',
+    resourceType: 'bigquery.googleapis.com/Dataset',
+    monitoredResource: 'bigquery_dataset',
+    name: 'projects/{project}/datasets/{resource.label.dataset_id}',
+});
+export const CLOUD_SCHEDULER_JOB_MAPPING = templateMapping({
+    nodeType: 'scheduledJob',
+    resourceType: 'cloudscheduler.googleapis.com/Job',
+    monitoredResource: 'cloud_scheduler_job',
+    name: 'projects/{project}/locations/{resource.label.location}/jobs/{resource.label.job_id}',
+});
+/**
+ * All the mappings of Cloud Monitoring monitored resources to the resources of nodes.
+ */
+export const MONITORING_RESOURCE_MAPPINGS = [
+    CLOUD_RUN_SERVICE_MAPPING,
+    PUBSUB_TOPIC_MAPPING,
+    PUBSUB_SUBSCRIPTION_MAPPING,
+    CLOUD_TASKS_QUEUE_MAPPING,
+    SPANNER_INSTANCE_MAPPING,
+    SPANNER_DATABASE_MAPPING,
+    FIRESTORE_DATABASE_MAPPING,
+    BIGQUERY_DATASET_MAPPING,
+    ...LOAD_BALANCER_BACKEND_MAPPINGS,
+    URL_MAP_MAPPING,
+    CLOUD_SCHEDULER_JOB_MAPPING,
+];
+/**
+ * Returns whether a subscription pushes messages to a service, rather than e.g. writing them to a table.
+ *
+ * @param node The node of the subscription.
+ * @param graph The graph containing the node.
+ * @returns `true` if the subscription targets a service.
+ */
+export function isPushSubscription(node, graph) {
+    return (graph.edges?.targets ?? []).some((edge) => edge.from === node.id && !!edge.to?.startsWith('service:'));
+}
+const CODE = 'code';
+const METHOD = 'method';
+/**
+ * The metrics read from Cloud Monitoring.
+ */
+export const MONITORING_METRICS = [
+    {
+        metric: 'requests',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/request_count',
+        groupBy: { [CODE]: 'metric.label.response_code_class' },
+    },
+    {
+        metric: 'latency',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/request_latencies',
+        groupBy: { [CODE]: 'metric.label.response_code_class' },
+    },
+    {
+        metric: 'instances',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/container/instance_count',
+        filter: 'metric.label.state = "active"',
+    },
+    {
+        metric: 'cpuUtilization',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/container/cpu/utilizations',
+        scale: 100,
+    },
+    {
+        metric: 'memoryUtilization',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/container/memory/utilizations',
+        scale: 100,
+    },
+    {
+        metric: 'concurrency',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'run.googleapis.com/container/max_request_concurrencies',
+        // Idle instances serve no request, and would pull the distribution down.
+        filter: 'metric.label.state = "active"',
+    },
+    {
+        metric: 'errorLogs',
+        mapping: CLOUD_RUN_SERVICE_MAPPING,
+        metricType: 'logging.googleapis.com/log_entry_count',
+        filter: 'metric.label.severity = one_of("ERROR", "CRITICAL", "ALERT", "EMERGENCY")',
+    },
+    {
+        metric: 'requests',
+        mapping: URL_MAP_MAPPING,
+        metricType: 'loadbalancing.googleapis.com/https/request_count',
+        groupBy: { [CODE]: 'metric.label.response_code_class' },
+        groupByNode: LOAD_BALANCER_BACKEND_MAPPINGS,
+    },
+    {
+        metric: 'latency',
+        mapping: URL_MAP_MAPPING,
+        metricType: 'loadbalancing.googleapis.com/https/total_latencies',
+        groupBy: { [CODE]: 'metric.label.response_code_class' },
+        groupByNode: LOAD_BALANCER_BACKEND_MAPPINGS,
+    },
+    {
+        metric: 'backendLatency',
+        mapping: URL_MAP_MAPPING,
+        metricType: 'loadbalancing.googleapis.com/https/backend_latencies',
+        groupBy: { [CODE]: 'metric.label.response_code_class' },
+        groupByNode: LOAD_BALANCER_BACKEND_MAPPINGS,
+    },
+    {
+        metric: 'published',
+        mapping: PUBSUB_TOPIC_MAPPING,
+        metricType: 'pubsub.googleapis.com/topic/message_sizes',
+        countRate: true,
+    },
+    {
+        metric: 'publishRequests',
+        mapping: PUBSUB_TOPIC_MAPPING,
+        metricType: 'pubsub.googleapis.com/topic/send_request_count',
+        groupBy: { [CODE]: 'metric.label.response_class' },
+    },
+    {
+        metric: 'publishLatency',
+        mapping: PUBSUB_TOPIC_MAPPING,
+        metricType: 'pubsub.googleapis.com/topic/send_request_latencies',
+        scale: 0.001,
+    },
+    {
+        metric: 'messageSize',
+        mapping: PUBSUB_TOPIC_MAPPING,
+        metricType: 'pubsub.googleapis.com/topic/message_sizes',
+    },
+    {
+        metric: 'backlog',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/num_undelivered_messages',
+    },
+    {
+        metric: 'backlogAge',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/oldest_unacked_message_age',
+        reducer: 'REDUCE_MAX',
+    },
+    {
+        metric: 'delivered',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/sent_message_count',
+    },
+    {
+        metric: 'acked',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/ack_message_count',
+        // Messages not matching the subscription's filter are acknowledged by Pub/Sub, without being delivered.
+        filter: 'metric.label.delivery_type != "filter"',
+    },
+    {
+        metric: 'pushRequests',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/push_request_count',
+        groupBy: { [CODE]: 'metric.label.response_class' },
+        measures: isPushSubscription,
+    },
+    {
+        metric: 'pushLatency',
+        mapping: PUBSUB_SUBSCRIPTION_MAPPING,
+        metricType: 'pubsub.googleapis.com/subscription/push_request_latencies',
+        groupBy: { [CODE]: 'metric.label.response_code' },
+        scale: 0.001,
+        measures: isPushSubscription,
+    },
+    {
+        metric: 'backlog',
+        mapping: CLOUD_TASKS_QUEUE_MAPPING,
+        metricType: 'cloudtasks.googleapis.com/queue/depth',
+    },
+    {
+        metric: 'published',
+        mapping: CLOUD_TASKS_QUEUE_MAPPING,
+        metricType: 'cloudtasks.googleapis.com/api/request_count',
+        filter: 'metric.label.api_method = "CreateTask"',
+        groupBy: { [CODE]: 'metric.label.response_code' },
+    },
+    {
+        metric: 'pushRequests',
+        mapping: CLOUD_TASKS_QUEUE_MAPPING,
+        metricType: 'cloudtasks.googleapis.com/queue/task_attempt_count',
+        groupBy: { [CODE]: 'metric.label.response_code' },
+    },
+    {
+        metric: 'dispatchDelay',
+        mapping: CLOUD_TASKS_QUEUE_MAPPING,
+        metricType: 'cloudtasks.googleapis.com/queue/task_attempt_delays',
+    },
+    {
+        metric: 'cpuUtilization',
+        mapping: SPANNER_INSTANCE_MAPPING,
+        metricType: 'spanner.googleapis.com/instance/cpu/utilization',
+        scale: 100,
+    },
+    {
+        metric: 'storageUtilization',
+        mapping: SPANNER_INSTANCE_MAPPING,
+        metricType: 'spanner.googleapis.com/instance/storage/utilization',
+        reducer: 'REDUCE_MAX',
+        scale: 100,
+    },
+    {
+        metric: 'nodes',
+        mapping: SPANNER_INSTANCE_MAPPING,
+        metricType: 'spanner.googleapis.com/instance/processing_units',
+        reducer: 'REDUCE_MAX',
+        scale: 0.001,
+    },
+    {
+        metric: 'requests',
+        mapping: SPANNER_DATABASE_MAPPING,
+        metricType: 'spanner.googleapis.com/api/api_request_count',
+        groupBy: {
+            [METHOD]: 'metric.label.method',
+            [CODE]: 'metric.label.status',
+        },
+    },
+    {
+        metric: 'latency',
+        mapping: SPANNER_DATABASE_MAPPING,
+        metricType: 'spanner.googleapis.com/api/request_latencies',
+        groupBy: { [METHOD]: 'metric.label.method' },
+        scale: 1000,
+    },
+    {
+        metric: 'storage',
+        mapping: SPANNER_DATABASE_MAPPING,
+        metricType: 'spanner.googleapis.com/instance/storage/used_bytes',
+    },
+    {
+        metric: 'cpuUtilization',
+        mapping: SPANNER_DATABASE_MAPPING,
+        metricType: 'spanner.googleapis.com/instance/cpu/utilization',
+        scale: 100,
+    },
+    {
+        metric: 'requests',
+        mapping: FIRESTORE_DATABASE_MAPPING,
+        metricType: 'firestore.googleapis.com/api/request_count',
+        groupBy: {
+            [METHOD]: 'metric.label.api_method',
+            [CODE]: 'metric.label.response_code',
+        },
+    },
+    {
+        metric: 'latency',
+        mapping: FIRESTORE_DATABASE_MAPPING,
+        metricType: 'firestore.googleapis.com/api/request_latencies',
+        groupBy: { [METHOD]: 'metric.label.api_method' },
+        scale: 1000,
+    },
+    {
+        metric: 'storage',
+        mapping: FIRESTORE_DATABASE_MAPPING,
+        metricType: 'firestore.googleapis.com/storage/data_and_index_storage_bytes',
+        // Sampled every minute, but written hours late, and in stretches separated by gaps of up to 13 hours.
+        minPeriod: 24 * 3600,
+    },
+    {
+        metric: 'storage',
+        mapping: BIGQUERY_DATASET_MAPPING,
+        metricType: 'bigquery.googleapis.com/storage/stored_bytes',
+        // Sampled every 30 minutes, and visible up to 3 hours later.
+        minPeriod: 4 * 3600,
+    },
+];
+/**
+ * The definitions of the metrics of the node types defined by the Google module.
+ */
+export const GOOGLE_GRAPH_METRIC_DEFINITIONS = {
+    'google.spanner.instance': {
+        cpuUtilization: {
+            name: 'CPU utilization',
+            description: 'CPU utilization of the instance, across all databases.',
+            kind: GraphMetricKind.Gauge,
+            unit: '%',
+        },
+        storageUtilization: {
+            name: 'Storage utilization',
+            description: 'Storage used, as a percentage of the limit of the provisioned capacity.',
+            kind: GraphMetricKind.Gauge,
+            unit: '%',
+        },
+        nodes: {
+            name: 'Nodes',
+            description: 'Provisioned compute capacity, where 1,000 processing units make a node.',
+            kind: GraphMetricKind.Gauge,
+            unit: '{node}',
+        },
+    },
+};
+/**
+ * Returns the kind of a metric read from Cloud Monitoring, from its definition for the type of its nodes.
+ *
+ * @param metric The metric.
+ * @returns The kind of the metric.
+ */
+export function monitoringMetricKind(metric) {
+    const { nodeType } = metric.mapping;
+    const definition = CORE_GRAPH_METRIC_DEFINITIONS[nodeType]?.[metric.metric] ??
+        GOOGLE_GRAPH_METRIC_DEFINITIONS[nodeType]?.[metric.metric];
+    if (!definition) {
+        throw new Error(`The metric '${metric.metric}' of node type '${nodeType}' has no definition.`);
+    }
+    return definition.kind;
+}

package/dist/graph/environment/monitoring-mappings.d.ts

@@ -0,0 +1,133 @@
+import { type GraphEnvironmentContext, type GraphNodeEntry } from '@causa/workspace-core/graph';
+/**
+ * A label of Cloud Monitoring time series or alerts, written as in `groupByFields`.
+ */
+export type MonitoringLabel = `resource.label.${string}` | `metric.label.${string}`;
+/**
+ * What identifies a resource in a time series or an alert.
+ */
+export type MonitoringResourceKey = {
+    /**
+     * The GCP project of the resource.
+     */
+    readonly project: string;
+    /**
+     * The values of all the key labels of the mapping, keyed by label.
+     */
+    readonly labels: Readonly<Record<MonitoringLabel, string>>;
+};
+/**
+ * Returns the full name of the resource identified by a key, or `undefined` if the key identifies no resource.
+ */
+export type MonitoringResourceResolver = (key: MonitoringResourceKey) => string | undefined;
+/**
+ * How time series and alerts on a Cloud Monitoring monitored resource identify the resources of the nodes of a type.
+ */
+export type MonitoringResourceMapping = {
+    /**
+     * The type of the nodes.
+     */
+    readonly nodeType: string;
+    /**
+     * The type of the nodes' resource, in their `data.resource.type`.
+     */
+    readonly resourceType: string;
+    /**
+     * The type of the monitored resource, e.g. `cloud_run_revision`.
+     */
+    readonly monitoredResource: string;
+    /**
+     * The labels identifying a resource within a project. A time series or alert missing one of them, or with an empty
+     * value, is not mapped.
+     */
+    readonly keyLabels: readonly MonitoringLabel[];
+    /**
+     * Prepares the resolution of keys to resource names in the environment.
+     * It is called through {@link prepareMonitoringMapping}, once per read or listing of alerts, and only when the mapping
+     * may apply. It throws a `GraphFactError` when a fact it depends on cannot be computed.
+     *
+     * @param environment The context of the environment.
+     * @returns The resolver.
+     */
+    resolver(environment: GraphEnvironmentContext): MonitoringResourceResolver | Promise<MonitoringResourceResolver>;
+};
+/**
+ * A mapping whose resolver is prepared.
+ */
+export type PreparedMonitoringMapping = {
+    /**
+     * The mapping.
+     */
+    readonly mapping: MonitoringResourceMapping;
+    /**
+     * The resolver of the mapping.
+     */
+    readonly resolve: MonitoringResourceResolver;
+};
+/**
+ * Returns the name of a label, without its source, e.g. `service_name` for `resource.label.service_name`.
+ *
+ * @param label The label.
+ * @returns The name of the label.
+ */
+export declare function monitoringLabelName(label: MonitoringLabel): string;
+/**
+ * Builds a mapping whose resource names are made of the values of its key labels.
+ *
+ * @param mapping The mapping, with the template of the name. `{project}` is the project of the key. Other placeholders
+ *   are labels, e.g. `projects/{project}/locations/{resource.label.location}/services/{resource.label.service_name}`,
+ *   and make up the key labels, in order.
+ * @returns The mapping.
+ */
+export declare function templateMapping(mapping: Pick<MonitoringResourceMapping, 'nodeType' | 'resourceType' | 'monitoredResource'> & {
+    readonly name: string;
+}): MonitoringResourceMapping;
+/**
+ * Reads the key of a mapping.
+ *
+ * @param mapping The mapping.
+ * @param project The project of the time series or alert.
+ * @param read Returns the value of a label of the time series or alert.
+ * @returns The key, or `undefined` if a key label is missing or empty.
+ */
+export declare function readMonitoringResourceKey(mapping: MonitoringResourceMapping, project: string, read: (label: MonitoringLabel) => string | undefined): MonitoringResourceKey | undefined;
+/**
+ * Prepares a mapping.
+ *
+ * @param environment The context of the environment.
+ * @param mapping The mapping.
+ * @returns The prepared mapping, or `undefined` if a fact its resolver depends on could not be computed. The failure is
+ *   reported by the fact.
+ */
+export declare function prepareMonitoringMapping(environment: GraphEnvironmentContext, mapping: MonitoringResourceMapping): Promise<PreparedMonitoringMapping | undefined>;
+/**
+ * A resource name resolved by a mapping, which no node of the graph holds.
+ */
+export type MissingMonitoredResource = {
+    /**
+     * The mapping that resolved the name.
+     */
+    readonly mapping: MonitoringResourceMapping;
+    /**
+     * The key from which the name was resolved.
+     */
+    readonly key: MonitoringResourceKey;
+    /**
+     * The resolved resource name.
+     */
+    readonly name: string;
+};
+/**
+ * Finds the node of the resource identified by a time series or an alert, trying mappings from the one with the most
+ * key labels. A mapping applies when its key can be read, and the name it resolves is held by a node of its node type.
+ *
+ * @param environment The context of the environment.
+ * @param mappings The prepared mappings that may apply.
+ * @param project The project of the time series or alert.
+ * @param read Returns the value of a label of the time series or alert.
+ * @returns The node, if a mapping applies, and the names resolved by mappings that are not held by a node of the graph.
+ */
+export declare function findMonitoredNode(environment: GraphEnvironmentContext, mappings: readonly PreparedMonitoringMapping[], project: string, read: (label: MonitoringLabel) => string | undefined): {
+    readonly node?: GraphNodeEntry;
+    readonly missing: MissingMonitoredResource[];
+};

package/dist/graph/environment/monitoring-mappings.js

@@ -0,0 +1,97 @@
+import { GraphFactError, } from '@causa/workspace-core/graph';
+/**
+ * Returns the name of a label, without its source, e.g. `service_name` for `resource.label.service_name`.
+ *
+ * @param label The label.
+ * @returns The name of the label.
+ */
+export function monitoringLabelName(label) {
+    return label.slice(label.lastIndexOf('.') + 1);
+}
+/**
+ * Builds a mapping whose resource names are made of the values of its key labels.
+ *
+ * @param mapping The mapping, with the template of the name. `{project}` is the project of the key. Other placeholders
+ *   are labels, e.g. `projects/{project}/locations/{resource.label.location}/services/{resource.label.service_name}`,
+ *   and make up the key labels, in order.
+ * @returns The mapping.
+ */
+export function templateMapping(mapping) {
+    const { name, ...rest } = mapping;
+    const placeholder = /\{([\w.]+)\}/g;
+    const keyLabels = [...name.matchAll(placeholder)]
+        .map(([, label]) => label)
+        .filter((label) => label !== 'project');
+    return {
+        ...rest,
+        keyLabels,
+        resolver: () => ({ project, labels }) => name.replace(placeholder, (_, label) => label === 'project' ? project : labels[label]),
+    };
+}
+/**
+ * Reads the key of a mapping.
+ *
+ * @param mapping The mapping.
+ * @param project The project of the time series or alert.
+ * @param read Returns the value of a label of the time series or alert.
+ * @returns The key, or `undefined` if a key label is missing or empty.
+ */
+export function readMonitoringResourceKey(mapping, project, read) {
+    const labels = {};
+    for (const label of mapping.keyLabels) {
+        const value = read(label);
+        if (!value) {
+            return undefined;
+        }
+        labels[label] = value;
+    }
+    return { project, labels: labels };
+}
+/**
+ * Prepares a mapping.
+ *
+ * @param environment The context of the environment.
+ * @param mapping The mapping.
+ * @returns The prepared mapping, or `undefined` if a fact its resolver depends on could not be computed. The failure is
+ *   reported by the fact.
+ */
+export async function prepareMonitoringMapping(environment, mapping) {
+    try {
+        return { mapping, resolve: await mapping.resolver(environment) };
+    }
+    catch (error) {
+        if (error instanceof GraphFactError) {
+            return undefined;
+        }
+        throw error;
+    }
+}
+/**
+ * Finds the node of the resource identified by a time series or an alert, trying mappings from the one with the most
+ * key labels. A mapping applies when its key can be read, and the name it resolves is held by a node of its node type.
+ *
+ * @param environment The context of the environment.
+ * @param mappings The prepared mappings that may apply.
+ * @param project The project of the time series or alert.
+ * @param read Returns the value of a label of the time series or alert.
+ * @returns The node, if a mapping applies, and the names resolved by mappings that are not held by a node of the graph.
+ */
+export function findMonitoredNode(environment, mappings, project, read) {
+    const missing = [];
+    const sorted = [...mappings].sort((a, b) => b.mapping.keyLabels.length - a.mapping.keyLabels.length);
+    for (const { mapping, resolve } of sorted) {
+        const key = readMonitoringResourceKey(mapping, project, read);
+        const name = key && resolve(key);
+        if (!key || !name) {
+            continue;
+        }
+        const node = environment.resourceNode(mapping.resourceType, name);
+        if (!node) {
+            missing.push({ mapping, key, name });
+        }
+        else if (node.type === mapping.nodeType) {
+            return { node, missing };
+        }
+    }
+    return { missing };
+}