@friggframework/core 2.0.0-next.11 → 2.0.0-next.110

package/syncs/repositories/sync-repository-postgres.js

@@ -0,0 +1,319 @@
+const { prisma } = require('../../database/prisma');
+const { SyncRepositoryInterface } = require('./sync-repository-interface');
+
+/**
+ * PostgreSQL Sync Repository Adapter
+ * Handles sync persistence using Prisma with PostgreSQL
+ *
+ * PostgreSQL-specific characteristics:
+ * - Uses implicit join tables for entity relations (_EntityToSync)
+ * - Uses Int IDs with autoincrement
+ * - Requires ID conversion: String (app layer) ↔ Int (database)
+ * - All returned IDs are converted to strings for application layer consistency
+ * - Uses connect/disconnect syntax for relations
+ */
+class SyncRepositoryPostgres extends SyncRepositoryInterface {
+    constructor() {
+        super();
+        this.prisma = prisma;
+    }
+
+    /**
+     * Convert string ID to integer for PostgreSQL queries
+     * @private
+     * @param {string|number|null|undefined} id - ID to convert
+     * @returns {number|null|undefined} Integer ID or null/undefined
+     * @throws {Error} If ID cannot be converted to integer
+     */
+    _convertId(id) {
+        if (id === null || id === undefined) return id;
+        const parsed = parseInt(id, 10);
+        if (isNaN(parsed)) {
+            throw new Error(`Invalid ID: ${id} cannot be converted to integer`);
+        }
+        return parsed;
+    }
+
+    /**
+     * Convert sync object IDs to strings
+     * @private
+     * @param {Object|null} sync - Sync object from database
+     * @returns {Object|null} Sync with string IDs
+     */
+    _convertSyncIds(sync) {
+        if (!sync) return sync;
+        return {
+            ...sync,
+            id: sync.id?.toString(),
+            integrationId: sync.integrationId?.toString(),
+            entities: sync.entities?.map(e => ({
+                ...e,
+                id: e.id?.toString(),
+                userId: e.userId?.toString(),
+                credentialId: e.credentialId?.toString()
+            })),
+            dataIdentifiers: sync.dataIdentifiers?.map(di => ({
+                ...di,
+                id: di.id?.toString(),
+                syncId: di.syncId?.toString(),
+                entityId: di.entityId?.toString(),
+                entity: di.entity ? {
+                    ...di.entity,
+                    id: di.entity.id?.toString(),
+                    userId: di.entity.userId?.toString(),
+                    credentialId: di.entity.credentialId?.toString()
+                } : di.entity
+            }))
+        };
+    }
+
+    /**
+     * Get a sync object by name, data identifier, and entity
+     *
+     * @param {string} name - The sync object name
+     * @param {Object} dataIdentifier - The data identifier object
+     * @param {string} entity - The entity ID (string from application layer)
+     * @returns {Promise<Object|null>} The sync object with string IDs or null
+     */
+    async getSyncObject(name, dataIdentifier, entity) {
+        const intEntityId = this._convertId(entity);
+        const syncList = await this.prisma.sync.findMany({
+            where: {
+                name,
+                dataIdentifiers: {
+                    some: {
+                        idData: dataIdentifier,
+                        entityId: intEntityId,
+                    },
+                },
+            },
+            include: {
+                entities: true,
+                dataIdentifiers: {
+                    include: {
+                        entity: true,
+                    },
+                },
+            },
+        });
+
+        if (syncList.length === 1) {
+            return this._convertSyncIds(syncList[0]);
+        } else if (syncList.length === 0) {
+            return null;
+        } else {
+            throw new Error(
+                `There are multiple sync objects with the name ${name}, for entities [${syncList[0].entities}] [${syncList[1].entities}]`
+            );
+        }
+    }
+
+    /**
+     * Create or update a sync object
+     *
+     * @param {Object} filter - Filter criteria for finding existing sync
+     * @param {Object} syncData - Sync data to create/update (with string IDs from application layer)
+     * @returns {Promise<Object>} The created or updated sync object with string IDs
+     */
+    async upsertSync(filter, syncData) {
+        // Find existing sync
+        const where = this._convertFilterToWhere(filter);
+        const existing = await this.prisma.sync.findFirst({ where });
+
+        // Convert IDs in syncData if present
+        const convertedData = { ...syncData };
+        if (convertedData.integrationId) {
+            convertedData.integrationId = this._convertId(convertedData.integrationId);
+        }
+
+        if (existing) {
+            // Update existing
+            const updated = await this.prisma.sync.update({
+                where: { id: existing.id },
+                data: convertedData,
+            });
+            return this._convertSyncIds(updated);
+        }
+
+        // Create new
+        const created = await this.prisma.sync.create({
+            data: convertedData,
+        });
+        return this._convertSyncIds(created);
+    }
+
+    /**
+     * Update a sync object by ID
+     *
+     * @param {string} id - The sync object ID (string from application layer)
+     * @param {Object} updates - Updates to apply (with string IDs from application layer)
+     * @returns {Promise<Object>} The updated sync object with string IDs
+     */
+    async updateSync(id, updates) {
+        const intId = this._convertId(id);
+
+        // Convert IDs in updates if present
+        const convertedUpdates = { ...updates };
+        if (convertedUpdates.integrationId) {
+            convertedUpdates.integrationId = this._convertId(convertedUpdates.integrationId);
+        }
+
+        const updated = await this.prisma.sync.update({
+            where: { id: intId },
+            data: convertedUpdates,
+        });
+        return this._convertSyncIds(updated);
+    }
+
+    /**
+     * Add a data identifier to a sync object
+     *
+     * @param {string} syncId - The sync object ID (string from application layer)
+     * @param {Object} dataIdentifier - The data identifier to add (with string entity ID)
+     * @returns {Promise<Object>} The updated sync object with string IDs
+     */
+    async addDataIdentifier(syncId, dataIdentifier) {
+        const intSyncId = this._convertId(syncId);
+        const intEntityId = this._convertId(dataIdentifier.entity);
+
+        // In Prisma, we create a new DataIdentifier record linked to the Sync
+        await this.prisma.dataIdentifier.create({
+            data: {
+                syncId: intSyncId,
+                entityId: intEntityId,
+                idData: dataIdentifier.id,
+                hash: dataIdentifier.hash,
+            },
+        });
+
+        // Return updated sync object
+        const sync = await this.prisma.sync.findUnique({
+            where: { id: intSyncId },
+            include: {
+                dataIdentifiers: true,
+            },
+        });
+        return this._convertSyncIds(sync);
+    }
+
+    /**
+     * Get entity object ID for entity ID from sync object
+     *
+     * This is a pure helper method (no database access)
+     *
+     * @param {Object} syncObj - The sync object (with string IDs from application layer)
+     * @param {string} entityId - The entity ID (string from application layer)
+     * @returns {Object} The entity object ID
+     */
+    getEntityObjIdForEntityIdFromObject(syncObj, entityId) {
+        if (!syncObj.dataIdentifiers) {
+            throw new Error('Sync object must include dataIdentifiers');
+        }
+
+        for (let dataIdentifier of syncObj.dataIdentifiers) {
+            // Compare string IDs (both should be strings at this point)
+            if (dataIdentifier.entityId === entityId) {
+                return dataIdentifier.idData;
+            }
+        }
+
+        throw new Error(
+            `Sync object ${syncObj.id} does not contain a data identifier for entity ${entityId}`
+        );
+    }
+
+    /**
+     * Find sync objects by filter
+     *
+     * @param {Object} filter - Filter criteria (with string IDs from application layer)
+     * @returns {Promise<Array>} Array of sync objects with string IDs
+     */
+    async findSyncs(filter) {
+        const where = this._convertFilterToWhere(filter);
+        const syncs = await this.prisma.sync.findMany({
+            where,
+            include: {
+                entities: true,
+                dataIdentifiers: {
+                    include: {
+                        entity: true,
+                    },
+                },
+            },
+        });
+        return syncs.map(sync => this._convertSyncIds(sync));
+    }
+
+    /**
+     * Find one sync object by filter
+     *
+     * @param {Object} filter - Filter criteria (with string IDs from application layer)
+     * @returns {Promise<Object|null>} The sync object with string IDs or null
+     */
+    async findOneSync(filter) {
+        const where = this._convertFilterToWhere(filter);
+        const sync = await this.prisma.sync.findFirst({
+            where,
+            include: {
+                entities: true,
+                dataIdentifiers: {
+                    include: {
+                        entity: true,
+                    },
+                },
+            },
+        });
+        return this._convertSyncIds(sync);
+    }
+
+    /**
+     * Delete a sync object by ID
+     *
+     * @param {string} id - The sync object ID (string from application layer)
+     * @returns {Promise<Object>} The deletion result with string IDs
+     */
+    async deleteSync(id) {
+        const intId = this._convertId(id);
+        // Prisma will cascade delete dataIdentifiers automatically
+        const deleted = await this.prisma.sync.delete({
+            where: { id: intId },
+        });
+        return this._convertSyncIds(deleted);
+    }
+
+    /**
+     * Convert Mongoose-style filter to Prisma where clause (converting IDs to Int)
+     * @private
+     * @param {Object} filter - Mongoose filter (with string IDs from application layer)
+     * @returns {Object} Prisma where clause (with Int IDs for PostgreSQL)
+     */
+    _convertFilterToWhere(filter) {
+        const where = {};
+
+        // Handle _id field (Mongoose uses _id, Prisma uses id)
+        if (filter._id) {
+            where.id = this._convertId(filter._id);
+        }
+
+        // Handle id field
+        if (filter.id) {
+            where.id = this._convertId(filter.id);
+        }
+
+        // Handle integrationId field
+        if (filter.integrationId) {
+            where.integrationId = this._convertId(filter.integrationId);
+        }
+
+        // Handle integration field (Mongoose uses integration, Prisma uses integrationId)
+        if (filter.integration) {
+            where.integrationId = this._convertId(filter.integration);
+        }
+
+        // Copy non-ID fields
+        const { _id, id, integrationId, integration, ...rest } = filter;
+        return { ...where, ...rest };
+    }
+}
+
+module.exports = { SyncRepositoryPostgres };

package/syncs/sync.js

@@ -1,5 +1,4 @@
 const md5 = require("md5");
-const ModuleManager = require('../module-plugin');
 const { debug } = require("packages/logs");
 const { get } = require("packages/assertions");
 

package/telemetry/README.md

@@ -0,0 +1,331 @@
+# Frigg Telemetry & Usage Tracking
+
+Vendor-neutral observability (traces + metrics) and durable, per-integration
+feature-usage counters for `@friggframework/core`, built on OpenTelemetry.
+Implements [ADR-011](../../../docs/architecture-decisions/011-integration-telemetry-and-usage-tracking.md).
+
+## Overview
+
+Two capabilities ride the same primitive:
+
+1. **Observability** — spans + low-cardinality metrics across handlers, API
+   modules, queues and webhooks, per integration, exported to any OTLP backend
+   (Honeycomb, Datadog, Grafana, an OTel Collector, …).
+2. **Usage tracking** — durable per-integration counters (records synced,
+   webhooks received, API requests, user actions, …) folded into a Frigg-owned
+   store that the reporting endpoint reads for apples-to-apples comparison.
+
+### Key properties
+
+- **Rides for free.** Framework seams are auto-instrumented — integrations get
+  handler/API-module/webhook metrics with zero code.
+- **No-op by default.** With no exporter configured the service emits nothing and
+  loads **zero** OpenTelemetry modules — no cold-start cost. Integration code can
+  always call `this.telemetry.*`.
+- **Vendor-neutral.** Integration code never imports a backend SDK. Swap exporters
+  in the app definition.
+- **Usage store ≠ APM.** Reports read the durable Frigg store, never an external
+  APM.
+
+## Configuration (app definition)
+
+```js
+// backend/index.js
+const Definition = {
+    name: 'my-app',
+    integrations: [HubSpotIntegration, SalesforceIntegration],
+
+    telemetry: {
+        // none | console | otlp | honeycomb | datadog
+        exporter: {
+            type: 'otlp',
+            endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+            // headers: { 'x-honeycomb-team': process.env.HONEYCOMB_KEY }, // or use type:'honeycomb' + apiKey
+        },
+        sampleRatio: 1.0, // parent-based trace sampling (0..1)
+        northStar: {
+            default: { name: 'records.synced' },
+        },
+    },
+};
+```
+
+**Exporter default (no `telemetry.exporter` set):** `console` only when
+`STAGE=local` (a genuinely local run); every deployed stage — including `dev` —
+defaults to `none` (no per-event cost, no data written to CloudWatch). Point
+`exporter` at an OTLP backend to turn export on.
+
+**Sampling (`sampleRatio`, `0..1`, default `1`):** the fraction of **traces**
+exported — a cost knob for high-traffic fleets (`0.1` ≈ keep 10%). Whole traces
+are sampled (trace-ID-based + parent-based, so a distributed trace is never
+half-kept), and it does **not** thin the durable **usage counters** — those stay
+exact at any ratio (they ride the event bus, not the sampled trace pipeline). It
+is **not** error-aware: a low ratio drops failed-run traces too, so for "keep all
+errors, sample the rest" use tail-based sampling at an OTel Collector, not this
+knob. Typical: `1.0` in dev, lower (e.g. `0.1`) in high-volume prod.
+
+### Environment variables
+
+| Variable | Purpose |
+| --- | --- |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP backend URL (referenced from `telemetry.exporter.endpoint`). Auto-passed through to Lambda when an OTLP-family exporter is configured. |
+| `OTEL_EXPORTER_OTLP_HEADERS` | OTLP headers (e.g. auth). |
+| `OTEL_FLUSH_TIMEOUT_MS` | Max time the handler waits to flush telemetry before returning (default `500`). Bounds tail latency if the backend is unreachable. |
+| `OTEL_METRIC_EXPORT_INTERVAL_MS` | Metric reader interval (default `60000`). |
+| `STAGE` | `local` enables the `console` default. |
+
+> **VPC note:** a Lambda in a private subnet needs a NAT gateway or VPC endpoint
+> to reach an external OTLP backend. Without egress the exporter fails silently
+> within `OTEL_FLUSH_TIMEOUT_MS`.
+
+## What you get for free (auto-instrumentation)
+
+No integration code required — every emission carries `{integration_type, event,
+status, ...}` bounded labels, with high-cardinality ids on span baggage / the bus
+context only.
+
+| Seam | Span | Metric |
+| --- | --- | --- |
+| Handler dispatch (USER_ACTION / CRON / QUEUE / WEBHOOK / lifecycle) | `frigg.handler.<type>` | `frigg.handler.invocations{integration_type, event, status}` |
+| Outbound API-module request | `frigg.apimodule.request` | `frigg.apimodule.requests{module, method, status}` |
+
+Request URLs are redacted (query string + userinfo stripped) before they touch a
+span, so credentials in query params never leak.
+
+> **Usage-attribution boundary.** The OTel metrics above fire for *every* seam
+> invocation. The durable per-integration **usage** rollup, though, only counts
+> emissions that carry an integration context — set by the handler seams. Requests
+> an API module makes *before an integration exists* (OAuth/token exchange, entity
+> discovery during connection setup) are observable in traces but not attributed to
+> an `api.requests` usage counter (there is no integration to attribute them to).
+
+## Custom metrics (integration code)
+
+Every integration instance carries `this.telemetry` (auto-tagged with its
+`integration_type`):
+
+```js
+class HubSpotIntegration extends IntegrationBase {
+    async deltaSync() {
+        // A span wraps the operation (nested under the handler span):
+        await this.telemetry.span('delta_sync', async () => {
+            const batch = await this.hubspot.api.getContacts();
+
+            // A counter — declare 'records.synced' in Definition.usage to persist it:
+            this.telemetry.count('records.synced', batch.length, {
+                entity: 'contact',
+            });
+
+            // A one-off event (attached to the active span):
+            this.telemetry.event('workflow_invoked', { workflow: 'lead_route' });
+        });
+    }
+}
+```
+
+## Usage counters (durable, comparable)
+
+### 1. Declare which counters an integration reports
+
+```js
+class HubSpotIntegration extends IntegrationBase {
+    static Definition = {
+        name: 'hubspot',
+        usage: {
+            // Canonical keys → comparable ACROSS integration types (reporting):
+            canonical: ['records.synced', 'webhooks.received', 'api.requests'],
+            // Custom keys → comparable only WITHIN this integration type:
+            custom: { 'deals.enriched': { unit: 'count', label: 'Deals enriched' } },
+        },
+    };
+}
+```
+
+**Canonical vocabulary** (core-owned, versioned):
+
+| Key | Source |
+| --- | --- |
+| `api.requests` | auto — every outbound API-module request |
+| `user_actions` | auto — every `USER_ACTION` handler |
+| `webhooks.received` | auto — the `ON_WEBHOOK` queue dispatch (per-integration, DB-connected) |
+| `records.synced` | explicit — `this.telemetry.count('records.synced', n, { entity })` |
+| `workflows.invoked` | explicit — `this.telemetry.count('workflows.invoked', 1, { workflow })` |
+
+Only **declared** keys are persisted. Declaring a canonical key opts the
+integration into the comparison report.
+
+### 2. Read the usage store
+
+`frigg.usage.*` (via `createFriggCommands`) reads the durable store — never an APM:
+
+```js
+const { createFriggCommands } = require('@friggframework/core');
+const frigg = createFriggCommands({ integrationClass: HubSpotIntegration });
+
+// Apples-to-apples comparison across integration types:
+await frigg.usage.getTotalsByDimension({
+    metric: 'records.synced',
+    groupBy: 'integrationType', // or 'metric'
+    since: daysAgo(30),
+    bucket: 'day', // 'day' (default) | 'hour'
+});
+// → [{ integrationType: 'hubspot', value: 4200 }, { integrationType: 'salesforce', value: 1180 }]
+
+// Trend series for one type (aggregated across its instances):
+await frigg.usage.getTimeSeries({
+    metric: 'records.synced',
+    integrationType: 'hubspot',
+    from: daysAgo(7),
+    to: new Date(),
+    bucket: 'day',
+});
+// → [{ bucket: 'day:2026-07-04', value: 610 }, { bucket: 'day:2026-07-05', value: 720 }]
+
+// Manual write (day + hour windows derived from `at`):
+await frigg.usage.recordUsageCounter({
+    integrationId: 'int_1',
+    integrationType: 'hubspot',
+    metric: 'deals.enriched',
+    value: 3,
+    at: new Date(),
+});
+```
+
+The [reporting endpoint](../reporting/README.md) surfaces these as additive
+`usage` columns on each `byType` bucket.
+
+## North Star metric
+
+Declare an adopter North Star that reports/snapshots read as a first-class
+counter — populated by direct emission, or derived from a trace signal with no
+integration code:
+
+```js
+telemetry: {
+    northStar: {
+        default: { name: 'records.synced' },
+        byType: {
+            hubspot: {
+                name: 'contacts_synced',
+                // derive from an auto-emitted signal:
+                deriveFrom: { apiRequest: { endpoint: '/contacts', method: 'POST' } },
+                // or: deriveFrom: { userAction: { action: 'route_lead' } }
+            },
+        },
+    },
+}
+```
+
+Read it as a first-class metric without knowing the configured key — the North
+Star resolves per integration type (`byType` wins over `default`):
+
+```js
+// The caller passes the North Star config it already holds (from the app
+// definition); this resolves the counter for the type and returns its totals.
+await frigg.usage.getNorthStarTotals({
+    northStar: definition.telemetry.northStar,
+    integrationType: 'hubspot',
+    since: daysAgo(30),
+});
+// → { metric: 'contacts_synced', totals: [{ integrationType: 'hubspot', value: 900 }] }
+// → null when `northStar` is absent or has no entry for the type
+```
+
+Or read it like any counter once you know the key:
+`frigg.usage.getTotalsByDimension({ metric: 'contacts_synced' })`; trends via `frigg.usage.getTimeSeries({ metric })`.
+
+> The North Star read takes its config as a **call argument** — nothing
+> telemetry-specific is threaded through `createFriggCommands`. The caller that
+> owns the app definition (e.g. a report runner) passes `telemetry.northStar` in;
+> omit it and `getNorthStarTotals()` returns `null`.
+
+## Plugin / extension tap
+
+Telemetry flows onto an internal event stream (independent of OTel export, so
+taps fire even with `exporter: none`). Two ways to subscribe:
+
+**Declarative (app definition)** — the framework wires these once per cold start,
+each guarded so a bad subscriber can't break emission or its siblings:
+
+```js
+// backend/index.js
+const Definition = {
+    telemetry: {
+        subscribers: [
+            // (a) declarative object — `event` optional; omit to receive both:
+            { event: 'metric', handler: ({ name, value, attributes, context }) => {
+                forwardToStatsd(name, value, attributes);
+            } },
+            // (b) factory — gets the telemetry service, registers itself, may
+            //     return an unsubscribe:
+            (telemetry) => telemetry.on('event', (payload) => auditSink.write(payload)),
+        ],
+    },
+};
+```
+
+**Imperative** — subscribe from anywhere that runs at startup:
+
+```js
+const { getTelemetry } = require('@friggframework/core');
+
+const off = getTelemetry().on('metric', ({ name, value, attributes, context }) => {
+    // `attributes` = bounded metric labels; `context` = high-cardinality ids
+    // (integrationId, integrationType, userId, url, …). Never throws upstream.
+});
+// off() to unsubscribe
+```
+
+> The bus payload shape (`{ name, value, attributes, context? }` for `'metric'`,
+> `{ name, attributes, context? }` for `'event'`) is a stable contract.
+
+## Cardinality rule
+
+High-cardinality identifiers (`integrationId`, `userId`, request `url`, action
+names) ride **span baggage / the bus `context`** — never OTel **metric** labels.
+Metric labels stay bounded (`integration_type`, `event`, `status`, `method`,
+`module`). The usage rollup derives per-integration counts from the bus context,
+not from metric labels.
+
+## How it works
+
+```
+this.telemetry.count / auto-instrumented seam
+        │  (bounded metric labels → OTel; ids → bus context via AsyncLocalStorage)
+        ├──────────────► OTel exporter (traces + metrics)   [observability]
+        └──────────────► TelemetryEventBus ('metric'/'event')
+                              │
+                              ├─ UsageRollupSubscriber ── buffers per invocation,
+                              │     flushes to the UsageCounter store on handler exit
+                              │     (discards on SQS redelivery — approximate contract)
+                              └─ your plugin taps
+
+frigg.usage.getTotalsByDimension / getTimeSeries ◄── UsageCounter store ──► reporting usage columns
+```
+
+- **Flush is Lambda-safe:** `create-handler` awaits a bounded `forceFlush` in a
+  `finally` (background timers can't fire once the container freezes).
+- **Usage accuracy is approximate:** at-least-once delivery means a retried
+  handler could double-count. The invocation buffer is discarded only when the
+  **whole** SQS batch is a redelivery (`ApproximateReceiveCount > 1`); a mixed
+  batch flushes so a redelivered sibling never drops a fresh record's counts.
+
+## Caveats / current limitations
+
+- **Usage persistence requires a DB-connected handler.** DB-free handlers (e.g.
+  the raw webhook-receipt route) can't write; `webhooks.received` is counted at
+  the DB-connected `ON_WEBHOOK` queue dispatch instead.
+- **DocumentDB** uses a raw-command adapter (`$runCommandRaw`) for increment and
+  aggregate; command shapes are unit-tested but not yet run against a real cluster.
+- **Retention:** the `UsageCounter` table has no pruning yet — hour-grain rows
+  accumulate. Add a scheduled prune for high-volume deployments. (Read paths are
+  covered by composite indexes `(metric, window)` and `(metric, integrationType,
+  window)`.)
+- Metric `value` is a `BigInt` per `(integrationId, integrationType, metric,
+  window)` row; reads coerce the sum to a JS Number (safe below 2^53).
+
+## See also
+
+- Architecture: [ADR-011](../../../docs/architecture-decisions/011-integration-telemetry-and-usage-tracking.md)
+- Reporting hand-off: [`reporting/README.md`](../reporting/README.md)
+- Encryption (same repository-triad pattern): [`database/encryption/README.md`](../database/encryption/README.md)