@friggframework/core 2.0.0-next.102 → 2.0.0-next.103

package/telemetry/telemetry-runtime.js

@@ -0,0 +1,147 @@
+// The process composition root for telemetry: one shared state, per-part lazy
+// getters, one atomic reset — replacing three separate singleton module caches
+// (telemetry / usage-rollup / plugin-subscribers) that previously had to be
+// reset together in the right order. Bus identity across the parts is guaranteed
+// by construction: the rollup and plugin subscribers attach to the same
+// getTelemetry() instance reached through this module's own closure.
+
+const { createTelemetry } = require('./telemetry-service');
+const { NoOpTelemetry } = require('./no-op-telemetry');
+
+const UNSET = Symbol('unset');
+
+function newState() {
+    return { telemetry: null, usageRollup: UNSET, pluginSubscribers: UNSET };
+}
+let state = newState();
+
+function loadTelemetryConfig() {
+    // Lazy-required to avoid a load-time cycle with the handlers layer.
+    const { loadAppDefinition } = require('../handlers/app-definition-loader');
+    return loadAppDefinition();
+}
+
+/**
+ * The process telemetry service. Created lazily on first use and reused for the
+ * life of the Lambda container, so the OTel SDK initialises at most once per cold
+ * start. Never throws — any failure falls back to the no-op.
+ */
+function getTelemetry() {
+    if (state.telemetry) return state.telemetry;
+    try {
+        const { telemetry = {} } = loadTelemetryConfig();
+        const stage = process.env.STAGE || process.env.NODE_ENV || 'production';
+        state.telemetry = createTelemetry({
+            exporter: telemetry.exporter,
+            sampleRatio: telemetry.sampleRatio,
+            resource: { service: process.env.FRIGG_STACK || 'frigg', stage },
+        });
+    } catch (_) {
+        state.telemetry = new NoOpTelemetry();
+    }
+    return state.telemetry;
+}
+
+/**
+ * The process usage-rollup subscriber, wired to getTelemetry()'s bus. Returns
+ * null when usage is disabled (no Definition.usage opt-in and no North Star) or
+ * the app definition can't load. Never throws — the warn lets operators tell
+ * "usage off" from "usage broken" (e.g. unresolved DB_TYPE).
+ */
+function getUsageRollupSubscriber() {
+    if (state.usageRollup !== UNSET) return state.usageRollup;
+    state.usageRollup = null;
+    try {
+        const { computeTrackedMetrics } = require('../usage/tracked-metrics');
+        const {
+            northStarKeys,
+            createNorthStarDerivationSubscriber,
+        } = require('./north-star');
+        const { integrations = [], telemetry = {} } = loadTelemetryConfig();
+
+        // Track both Definition.usage opt-ins and any North Star counter keys,
+        // so a derived North Star persists even without an explicit usage decl.
+        const trackedMetrics = computeTrackedMetrics(integrations);
+        for (const key of northStarKeys(telemetry.northStar)) {
+            trackedMetrics.add(key);
+        }
+        if (trackedMetrics.size === 0) return state.usageRollup;
+
+        const {
+            createUsageRepository,
+        } = require('../usage/repositories/usage-repository-factory');
+        const {
+            createUsageRollupSubscriber,
+        } = require('./usage-rollup-subscriber');
+        const telemetryService = getTelemetry(); // same memo → shared bus
+
+        state.usageRollup = createUsageRollupSubscriber({
+            telemetry: telemetryService,
+            usageRepository: createUsageRepository(),
+            trackedMetrics,
+        });
+
+        // North Star derived-from-trace: emits the North Star counter when a
+        // configured signal matches; the emission flows to the rollup above.
+        if (telemetry.northStar) {
+            createNorthStarDerivationSubscriber({
+                telemetry: telemetryService,
+                northStar: telemetry.northStar,
+            });
+        }
+    } catch (error) {
+        console.warn(
+            `[Frigg][usage] rollup disabled: ${error && error.message}`
+        );
+        state.usageRollup = null;
+    }
+    return state.usageRollup;
+}
+
+/**
+ * Adopter-declared telemetry subscribers, wired to getTelemetry()'s bus once per
+ * cold start. Independent of the usage rollup (adopter taps wire even when no
+ * integration declares Definition.usage). Never throws.
+ */
+function getPluginTelemetrySubscribers() {
+    if (state.pluginSubscribers !== UNSET) return state.pluginSubscribers;
+    state.pluginSubscribers = [];
+    try {
+        const { wireTelemetrySubscribers } = require('./plugin-subscribers');
+        const { telemetry = {} } = loadTelemetryConfig();
+        const subscribers = telemetry.subscribers || [];
+        if (subscribers.length === 0) return state.pluginSubscribers;
+
+        state.pluginSubscribers = wireTelemetrySubscribers({
+            telemetry: getTelemetry(),
+            subscribers,
+        });
+    } catch (error) {
+        console.warn(
+            `[Frigg][telemetry] plugin subscribers disabled: ${
+                error && error.message
+            }`
+        );
+        state.pluginSubscribers = [];
+    }
+    return state.pluginSubscribers;
+}
+
+/** Test-only: reset the whole runtime — no stale cross-memo states possible. */
+function resetTelemetryRuntimeForTests() {
+    state = newState();
+}
+
+/** Test-only: install a telemetry service; dependent memos clear atomically. */
+function setTelemetryForTests(telemetry) {
+    state = newState();
+    state.telemetry = telemetry;
+}
+
+module.exports = {
+    getTelemetry,
+    getUsageRollupSubscriber,
+    getPluginTelemetrySubscribers,
+    resetTelemetryRuntimeForTests,
+    setTelemetryForTests,
+};

package/telemetry/telemetry-service.js

@@ -0,0 +1,51 @@
+const { NoOpTelemetry } = require('./no-op-telemetry');
+const { createTelemetryEventBus } = require('./telemetry-event-bus');
+
+/**
+ * @typedef {object} TelemetryService The vendor-neutral telemetry port. Both
+ * NoOpTelemetry and OtelTelemetry implement it (duck-typed; the bound wrapper
+ * from bindTelemetryContext is a plain object that also satisfies it):
+ *   count(name, value?, attributes?, context?) · event(name, attributes?, context?)
+ *   span(name, fn) · startSpan(name?) · withContext(context, fn)
+ *   on(eventType, cb) · forceFlush() · shutdown() · isEnabled()
+ */
+
+/**
+ * Determine whether an exporter descriptor means "emit nothing".
+ * Absent config or `{ type: 'none' }` (or `'noop'`) → no-op.
+ */
+function isNoOpExporter(exporter) {
+    if (!exporter) return true;
+    const type = exporter.type;
+    return !type || type === 'none' || type === 'noop';
+}
+
+/**
+ * Create the vendor-neutral telemetry service — the adapter selector. Integration
+ * code uses the returned instance (`this.telemetry`) and never imports a backend
+ * SDK. With no exporter configured it returns `NoOpTelemetry` (loads zero OTel);
+ * a real exporter lazily constructs `OtelTelemetry`.
+ *
+ * @param {object} [options]
+ * @param {object} [options.exporter] Exporter descriptor, e.g. `{ type: 'otlp', endpoint }`.
+ * @param {object} [options.resource] Resource attributes, e.g. `{ service, stage }`.
+ * @param {number} [options.sampleRatio] Parent-based sampling ratio (0..1).
+ * @param {object} [options.bus] Event bus to reuse (defaults to a fresh one).
+ */
+function createTelemetry(options = {}) {
+    // The internal event stream is always on and independent of OTel export:
+    // the usage rollup + plugin taps must work even when
+    // the OTel exporter is a no-op. The bus is pure JS, so this does not load
+    // any OTel module on the no-op path.
+    const bus = options.bus || createTelemetryEventBus();
+
+    if (isNoOpExporter(options.exporter)) {
+        return new NoOpTelemetry({ bus });
+    }
+
+    // Lazy-required so the no-op path never loads the OTel SDK.
+    const { OtelTelemetry } = require('./otel-telemetry');
+    return new OtelTelemetry({ ...options, bus });
+}
+
+module.exports = { createTelemetry, isNoOpExporter };

package/telemetry/usage-rollup-subscriber.js

@@ -0,0 +1,116 @@
+const { computeUsageWindows } = require('../usage/usage-windows');
+
+const WEBHOOK_EVENT_NAMES = new Set(['ON_WEBHOOK']);
+
+/**
+ * Framework auto-signal metric names → the canonical usage key they feed.
+ * Resolvers receive (attributes, context) and may return null to
+ * decline. Handler invocations map by event: USER_ACTION → user_actions; the
+ * DB-connected `ON_WEBHOOK` queue dispatch → webhooks.received (per-integration,
+ * and where a durable write is actually possible — the HTTP receipt handler is
+ * DB-free so its buffer is discarded).
+ */
+const METRIC_TO_CANONICAL = {
+    'frigg.apimodule.requests': () => 'api.requests',
+    'frigg.handler.invocations': (attrs, ctx) => {
+        if (attrs && attrs.event === 'USER_ACTION') return 'user_actions';
+        if (ctx && WEBHOOK_EVENT_NAMES.has(ctx.event_name)) {
+            return 'webhooks.received';
+        }
+        return null;
+    },
+};
+
+/**
+ * The built-in usage-rollup subscriber. Subscribes to the
+ * telemetry event bus and folds *declared* counters (canonical or custom) into
+ * the durable usage store. It buffers within an invocation and writes on
+ * `flush()` (no timers — Lambda-safe), or drops the buffer on `discard()` for an
+ * SQS redelivery so at-least-once delivery does not double-count (approximate
+ * accuracy contract).
+ *
+ * Attribution: high-cardinality ids never ride metric labels, so integrationId
+ * comes from the bus-only `context` (populated from the ambient handler context)
+ * and otherwise falls back to the bounded integration_type.
+ */
+function createUsageRollupSubscriber({
+    telemetry,
+    usageRepository,
+    trackedMetrics = new Set(),
+    now = () => new Date(),
+}) {
+    // Buffer key is a JSON tuple [integrationId, integrationType, metric, window]
+    // — collision-proof for developer-defined custom keys / names.
+    let buffer = new Map();
+
+    function resolveUsageKey(name, attributes, context) {
+        if (trackedMetrics.has(name)) return name;
+        const resolver = METRIC_TO_CANONICAL[name];
+        const canonical = resolver ? resolver(attributes || {}, context) : null;
+        return canonical && trackedMetrics.has(canonical) ? canonical : null;
+    }
+
+    function onMetric(payload) {
+        try {
+            const { name, value = 1, attributes = {}, context } = payload;
+            const metric = resolveUsageKey(name, attributes, context);
+            if (!metric) return;
+
+            const integrationType =
+                context?.integrationType || attributes.integration_type || null;
+            if (!integrationType) return; // cannot attribute — skip
+
+            const integrationId = context?.integrationId || integrationType;
+
+            for (const window of computeUsageWindows(now())) {
+                const key = JSON.stringify([
+                    integrationId,
+                    integrationType,
+                    metric,
+                    window,
+                ]);
+                buffer.set(key, (buffer.get(key) || 0) + value);
+            }
+        } catch (_) {
+            // Rollup must never break emission.
+        }
+    }
+
+    const unsubscribe = telemetry.on('metric', onMetric);
+
+    async function flush() {
+        if (buffer.size === 0) return;
+        const pending = buffer;
+        buffer = new Map();
+        for (const [key, value] of pending) {
+            const [integrationId, integrationType, metric, window] =
+                JSON.parse(key);
+            try {
+                await usageRepository.increment({
+                    integrationId,
+                    integrationType,
+                    metric,
+                    window,
+                    value,
+                });
+            } catch (_) {
+                // A single failed write must not abort the rest of the flush.
+            }
+        }
+    }
+
+    function discard() {
+        buffer = new Map();
+    }
+
+    return {
+        flush,
+        discard,
+        unsubscribe,
+        get bufferSize() {
+            return buffer.size;
+        },
+    };
+}
+
+module.exports = { createUsageRollupSubscriber, METRIC_TO_CANONICAL };

package/usage/README.md

@@ -0,0 +1,54 @@
+# Usage Store
+
+The durable, Frigg-owned store behind ADR-011 feature-usage tracking. Holds
+per-integration counters that the reporting endpoint reads for cross-integration
+comparison and trend series.
+
+Populated by the telemetry usage rollup and read via `frigg.usage.*` — see
+[`telemetry/README.md`](../telemetry/README.md) for the full guide, config, and
+usage examples. This README covers the store internals only.
+
+## Fact row
+
+`UsageCounter { integrationId, integrationType, metric, window, value, updatedAt }`,
+uniquely keyed by `(integrationId, integrationType, metric, window)`. `window` is
+`day:YYYY-MM-DD` or `hour:YYYY-MM-DDTHH` (UTC).
+
+## Isolation (ADR-010 Decision 3)
+
+The store is deliberately isolated from user/integration-scoped data:
+
+- **No `userId`** and **no foreign key** to `Integration` — a user-scoped query
+  can never return a usage row, and usage history survives integration deletion.
+- **Not** in the encryption registry — dimensions are bounded and non-sensitive.
+
+## Repository triad
+
+Mirrors the reporting/process pattern — an interface plus PostgreSQL, MongoDB and
+DocumentDB adapters selected by `DB_TYPE`:
+
+```js
+const { createUsageRepository } = require('@friggframework/core'); // usage-repository-factory
+
+class UsageRepositoryInterface {
+    async increment({ integrationId, integrationType, metric, window, value }) {} // atomic upsert
+    async getTotalsByDimension({ metric, groupBy, since, bucket }) {}   // comparison (one window granularity)
+    async getTimeSeries({ metric, integrationType, from, to, bucket }) {} // trend (aggregated across instances)
+}
+```
+
+- **`increment`** is atomic: PostgreSQL uses Prisma `upsert` with
+  `{ value: { increment } }`; a concurrent first-insert race (`P2002`) retries
+  once onto the atomic update path.
+- **`getTotalsByDimension`** filters to a single window granularity (default `day`) so day and
+  hour rows are never double-summed. Requires a `metric`; `groupBy` is
+  allow-listed to `integrationType` / `metric`.
+- **`getTimeSeries`** aggregates across integration instances (`groupBy(window) + sum`)
+  and range-filters on the window key. Requires an `integrationType`.
+
+> **DocumentDB:** the adapter overrides increment/getTotalsByDimension/getTimeSeries with raw commands
+> (`$runCommandRaw`: a `$inc` upsert via `documentdb-utils.updateOne`, and a
+> cursor-drained `$aggregate` `$group/$sum`) — matching every other DocumentDB
+> adapter, since Prisma's Mongo engine emits upsert/groupBy shapes DocumentDB
+> rejects and cursor reads truncate at ~101 docs. Command shapes are unit-tested;
+> run an end-to-end check against a real cluster before GA.

package/usage/index.js

@@ -0,0 +1,17 @@
+const {
+    createUsageRepository,
+    UsageRepositoryPrisma,
+    UsageRepositoryDocumentDB,
+} = require('./repositories/usage-repository-factory');
+const {
+    UsageRepositoryInterface,
+} = require('./repositories/usage-repository-interface');
+const { computeTrackedMetrics } = require('./tracked-metrics');
+
+module.exports = {
+    createUsageRepository,
+    computeTrackedMetrics,
+    UsageRepositoryInterface,
+    UsageRepositoryPrisma,
+    UsageRepositoryDocumentDB,
+};

package/usage/repositories/usage-repository-documentdb.js

@@ -0,0 +1,194 @@
+const { prisma } = require('../../database/prisma');
+const { updateOne } = require('../../database/documentdb-utils');
+const { UsageRepositoryPrisma } = require('./usage-repository-prisma');
+const { windowKey } = require('../usage-windows');
+
+const COLLECTION = 'UsageCounter';
+const DRAIN_BATCH_SIZE = 1000;
+const MAX_BATCHES = 100000;
+const VALID_GROUP_BY = new Set(['integrationType', 'metric']);
+const VALID_BUCKETS = new Set(['day', 'hour']);
+
+/**
+ * DocumentDB usage store. Amazon DocumentDB does not accept the command shapes
+ * Prisma's Mongo engine emits for `upsert({ update: { value: { increment } } })`
+ * and `groupBy({ _sum })`, and cursor reads truncate at ~101 docs — so, like
+ * every other DocumentDB adapter in this repo, these operations are issued as
+ * raw commands (`$runCommandRaw`) via the validated documentdb-utils helpers and
+ * a drained aggregate cursor. No timestamps are managed: getTotalsByDimension/getTimeSeries filter on
+ * the window KEY (not write-time), so createdAt/updatedAt are unnecessary here.
+ */
+class UsageRepositoryDocumentDB extends UsageRepositoryPrisma {
+    constructor() {
+        super();
+        this.prisma = prisma;
+    }
+
+    async increment({ integrationId, integrationType, metric, window, value = 1 }) {
+        // Atomic upsert-increment: $inc creates the field at the increment value
+        // on insert; $setOnInsert stamps the identity on first write. The compound
+        // filter is the unique key, so concurrent writers converge on one row.
+        await updateOne(
+            this.prisma,
+            COLLECTION,
+            { integrationId, integrationType, metric, window },
+            {
+                $inc: { value },
+                $setOnInsert: {
+                    integrationId,
+                    integrationType,
+                    metric,
+                    window,
+                },
+            },
+            { upsert: true }
+        );
+    }
+
+    async getTotalsByDimension({
+        metric,
+        groupBy = 'integrationType',
+        since,
+        bucket = 'day',
+    } = {}) {
+        if (!metric) {
+            throw new Error('getTotalsByDimension requires a metric (units are per-metric)');
+        }
+        assertGroupBy(groupBy);
+        assertBucket(bucket);
+
+        // Single window granularity (day: OR hour:) — never sum across both.
+        // `since` bounds on the window KEY (mirrors getTimeSeries / the Prisma adapter).
+        const rows = await this._aggregateDrained([
+            {
+                $match: {
+                    metric,
+                    window: windowMatch(bucket, {
+                        gte: since ? windowKey(bucket, since) : undefined,
+                    }),
+                },
+            },
+            { $group: { _id: `$${groupBy}`, value: { $sum: '$value' } } },
+        ]);
+
+        return rows.map((row) => ({
+            [groupBy]: row._id,
+            value: toNumber(row.value),
+        }));
+    }
+
+    async getTimeSeries({ metric, integrationType, from, to, bucket = 'day' } = {}) {
+        assertBucket(bucket);
+        if (!integrationType) {
+            throw new Error('getTimeSeries requires an integrationType');
+        }
+
+        const rows = await this._aggregateDrained([
+            {
+                $match: {
+                    metric,
+                    integrationType,
+                    window: windowMatch(bucket, {
+                        gte: from ? windowKey(bucket, from) : undefined,
+                        lte: to ? windowKey(bucket, to) : undefined,
+                    }),
+                },
+            },
+            { $group: { _id: '$window', value: { $sum: '$value' } } },
+            { $sort: { _id: 1 } },
+        ]);
+
+        return rows.map((row) => ({
+            bucket: row._id,
+            value: toNumber(row.value),
+        }));
+    }
+
+    async _aggregateDrained(pipeline) {
+        const first = await this.prisma.$runCommandRaw({
+            aggregate: COLLECTION,
+            pipeline,
+            cursor: { batchSize: DRAIN_BATCH_SIZE },
+        });
+        return drainCursor(this.prisma, COLLECTION, first);
+    }
+}
+
+/** A `$match` window clause: prefix by granularity, optionally range-bounded. */
+function windowMatch(bucket, { gte, lte } = {}) {
+    const clause = { $regex: `^${bucket}:` };
+    if (gte) clause.$gte = gte;
+    if (lte) clause.$lte = lte;
+    return clause;
+}
+
+/**
+ * Coerce an aggregate `$sum` result to a JS Number. Prisma $runCommandRaw returns
+ * extended JSON, so a 64-bit sum can arrive as { $numberLong: "..." } (or
+ * $numberInt/$numberDouble); counts never approach 2^53.
+ */
+function toNumber(value) {
+    if (value === null || value === undefined) return 0;
+    if (typeof value === 'number') return value;
+    if (typeof value === 'bigint') return Number(value);
+    if (typeof value === 'object') {
+        const raw =
+            value.$numberLong ?? value.$numberInt ?? value.$numberDouble;
+        if (raw !== undefined) return Number(raw);
+    }
+    return Number(value) || 0;
+}
+
+async function drainCursor(client, collection, firstResult) {
+    const cursor = firstResult?.cursor || {};
+    const docs = [...(cursor.firstBatch || [])];
+    let cursorId = cursor.id;
+    let batches = 0;
+
+    while (isCursorOpen(cursorId) && batches < MAX_BATCHES) {
+        batches += 1;
+        const next = await client.$runCommandRaw({
+            getMore: cursorId,
+            collection,
+            batchSize: DRAIN_BATCH_SIZE,
+        });
+        const nextCursor = next?.cursor || {};
+        const nextBatch = nextCursor.nextBatch || [];
+        docs.push(...nextBatch);
+        cursorId = nextCursor.id;
+        if (nextBatch.length === 0) break;
+    }
+    return docs;
+}
+
+function isCursorOpen(id) {
+    if (id === undefined || id === null) return false;
+    if (typeof id === 'number') return id !== 0;
+    if (typeof id === 'bigint') return id !== 0n;
+    if (typeof id === 'object' && id.$numberLong !== undefined) {
+        return id.$numberLong !== '0';
+    }
+    return String(id) !== '0';
+}
+
+function assertGroupBy(groupBy) {
+    if (!VALID_GROUP_BY.has(groupBy)) {
+        throw new Error(
+            `Invalid groupBy "${groupBy}". Allowed: ${[...VALID_GROUP_BY].join(
+                ', '
+            )}`
+        );
+    }
+}
+
+function assertBucket(bucket) {
+    if (!VALID_BUCKETS.has(bucket)) {
+        throw new Error(
+            `Invalid bucket "${bucket}". Allowed: ${[...VALID_BUCKETS].join(
+                ', '
+            )}`
+        );
+    }
+}
+
+module.exports = { UsageRepositoryDocumentDB };

package/usage/repositories/usage-repository-factory.js

@@ -0,0 +1,25 @@
+const { UsageRepositoryPrisma } = require('./usage-repository-prisma');
+const { UsageRepositoryDocumentDB } = require('./usage-repository-documentdb');
+const config = require('../../database/config');
+
+function createUsageRepository() {
+    const dbType = config.DB_TYPE;
+
+    switch (dbType) {
+        case 'mongodb':
+        case 'postgresql':
+            return new UsageRepositoryPrisma();
+        case 'documentdb':
+            return new UsageRepositoryDocumentDB();
+        default:
+            throw new Error(
+                `Unsupported database type: ${dbType}. Supported values: 'mongodb', 'documentdb', 'postgresql'`
+            );
+    }
+}
+
+module.exports = {
+    createUsageRepository,
+    UsageRepositoryPrisma,
+    UsageRepositoryDocumentDB,
+};

package/usage/repositories/usage-repository-interface.js

@@ -0,0 +1,37 @@
+/**
+ * Port for the durable usage-counter store.
+ *
+ * The store is deliberately isolated (ADR-010 Decision 3): its fact rows carry
+ * NO userId and NO foreign key to Integration, so a user-scoped query can never
+ * return a usage row and usage history survives integration deletion.
+ *
+ * Fact row: { integrationId, integrationType, metric, window, value, updatedAt }
+ * Dimensions must be bounded — high-cardinality ids stay on traces.
+ */
+class UsageRepositoryInterface {
+    /**
+     * Atomically add `value` to the counter for
+     * (integrationId, integrationType, metric, window), inserting if absent.
+     */
+    async increment(/* { integrationId, integrationType, metric, window, value } */) {
+        throw new Error('increment must be implemented by subclass');
+    }
+
+    /**
+     * Sum a metric grouped by a bounded dimension since an optional timestamp.
+     * @returns {Promise<Array<{[groupBy]: string, value: number}>>}
+     */
+    async getTotalsByDimension(/* { metric, groupBy, since } */) {
+        throw new Error('getTotalsByDimension must be implemented by subclass');
+    }
+
+    /**
+     * Time series of a metric for one integration type.
+     * @returns {Promise<Array<{bucket: string, value: number}>>}
+     */
+    async getTimeSeries(/* { metric, integrationType, from, to, bucket } */) {
+        throw new Error('getTimeSeries must be implemented by subclass');
+    }
+}
+
+module.exports = { UsageRepositoryInterface };