@friggframework/core 2.0.0-next.111 → 2.0.0-next.113

package/.eslintrc.json

@@ -1,3 +1,20 @@
 {
-    "extends": "@friggframework/eslint-config"
+    "extends": "@friggframework/eslint-config",
+    "rules": {
+        "no-restricted-syntax": [
+            "warn",
+            {
+                "selector": "CallExpression[callee.object.object.name='process'][callee.object.property.name=/^(stdout|stderr)$/][callee.property.name='write']",
+                "message": "Write through the logger; only logs/sinks.js writes to fd 1 (ADR-048 §14)"
+            }
+        ]
+    },
+    "overrides": [
+        {
+            "files": ["*.test.js", "database/utils/prisma-runner.js"],
+            "rules": {
+                "no-restricted-syntax": "off"
+            }
+        }
+    ]
 }

package/CLAUDE.md

@@ -211,7 +211,56 @@ packages/core/
 - `integration-repository-factory.js` - Creates database-specific repositories
 - `integration-repository-mongo.js` - MongoDB implementation
 - `integration-repository-postgres.js` - PostgreSQL implementation
-- `integration-mapping-repository-*.js` - Mapping data persistence
+- `integration-mapping-repository-*.js` - Mapping data persistence.
+  `queryMappings(integrationId, { where, orderBy, skip, take, omit })`
+  returns `{ mappings, total }`: one filtered, ordered page of an
+  integration's mappings, for callers that cannot load every row through
+  `findMappingsByIntegration`. `where` is an array of ANDed conditions
+  (at most 20); an entry may be `{ anyOf: [...] }`, ORed, one level deep.
+  Conditions are `{ path: 'mapping.<segment>...', op: 'exists' | 'notExists' }`
+  (JSON null counts as absent), `{ path: 'mapping.…', op: 'in', value: string[] }`
+  (1–500 strings) and `{ path: 'sourceId', op: 'notStartsWith', value }`
+  (a NULL sourceId matches). `orderBy` is `{ path: 'mapping.…', direction:
+  'asc' | 'desc' }`, nulls last, ties broken by id in the same direction;
+  without it rows come in id order. `take` is 1–500. `omit` lists top-level
+  mapping keys to leave out of the rows; never write such rows back. Path
+  segments must match `^[A-Za-z_][A-Za-z0-9_]*$`. The PostgreSQL, MongoDB and
+  DocumentDB adapters give the same pages and totals;
+  `integration-mapping-repository-query-parity.test.js` checks that against
+  real databases when `QUERY_MAPPINGS_PARITY_MONGO_URL` /
+  `QUERY_MAPPINGS_PARITY_POSTGRES_URL` are set. Two orderings still differ:
+  strings compare by the database collation on PostgreSQL and by code point
+  on MongoDB and DocumentDB, and arrays or objects at the sort path order
+  among themselves only on PostgreSQL (the other adapters fall back to id).
+  The legacy `IntegrationMappingRepository` inherits the port's
+  "not supported by this database adapter yet" error. Every adapter refuses
+  to run while field-level encryption still encrypts
+  `IntegrationMapping.mapping` on write (see `database/encryption/README.md`),
+  and returns rows decrypted like `findMappingsByIntegration`. PostgreSQL and
+  MongoDB decrypt through `decryptQueriedMappings`
+  (`database/encryption/integration-mapping-encryption.js`), a no-op while
+  encryption is off or lists no `IntegrationMapping` field besides `mapping`.
+  Validation lives in `integration-mapping-query.js`; a new operator is one
+  entry in its `OPERATORS` table, one in the Postgres adapter's
+  `CONDITION_SQL` and one in `CONDITION_EXPRESSIONS` in
+  `integration-mapping-query-pipeline.js`, the aggregation stages both
+  MongoDB-protocol adapters share. Those stages may only use what Amazon
+  DocumentDB 4.0 and 5.0 support (no `$facet`, `$getField`, `$set` or
+  `$unset`).
+  **Cost**: PostgreSQL answers in one SQL statement, which reads every row
+  of the integration and evaluates the JSON paths per row, because no JSON
+  index exists. With ~4 KB mappings on PostgreSQL 16 that is about 0.3 s per
+  call at 10⁴ rows per integration and 2.5–3 s at 10⁵. MongoDB answers in one
+  aggregate: the `integrationId` index narrows `$match` to the integration,
+  the `$expr` is then evaluated per document, and `$facet` returns the page
+  with its `$count`, so a page (after `omit`) must fit the 16 MB document
+  limit. DocumentDB has no `$facet`: the page and the count are two
+  concurrent aggregates, so the total does not come from the same snapshot
+  as the page. Both sort in memory on a computed key (`allowDiskUse`). With
+  ~4 KB mappings on MongoDB 7 that is about 0.05–0.1 s per call at 10⁴ rows
+  per integration and 0.5–0.7 s at 10⁵ (the DocumentDB pipeline, run on
+  MongoDB, 0.6–0.8 s). On every adapter a deep offset or an empty page past
+  the end costs about the same as the first page.
 - `process-repository-*.js` - Process (long-running job) persistence.
   Implements `applyProcessUpdate(processId, ops)` — a race-safe alternative
   to `update(id, patch)` that routes increments, sets, and bounded-array
@@ -414,6 +463,43 @@ const decrypted = await cryptor.decrypt(encrypted);
 - `integration-event-dispatcher.js` - Routes events to integration handlers
 - Supports lifecycle events and user actions
 
+**Queue handler delivery**: a handler dispatched from the integration queue
+(any `this.events` entry reached through SQS, including `ON_WEBHOOK` and
+scheduled jobs) receives `{ data, context, delivery }`. HTTP-dispatched
+handlers (`{ req, res, next }`) and `this.on` events do not.
+
+| Field | Type | Value |
+|---|---|---|
+| `delivery.receiveCount` | `number \| undefined` | SQS `ApproximateReceiveCount` of this delivery |
+| `delivery.maxReceiveCount` | `number \| undefined` | Receives allowed before SQS moves the message to the DLQ (`FRIGG_QUEUE_MAX_RECEIVE_COUNT`) |
+| `delivery.isLastAttempt` | `boolean` | `true` only when both counts are known and `receiveCount >= maxReceiveCount` |
+
+`isLastAttempt` is `false` when either count is unknown: a local or non-SQS
+invocation, or a queue whose redrive policy the stack does not own
+(`ownership.queue: 'external'`). The value is information only: core still
+rethrows retryable errors and discards halt errors (4xx except 408/429).
+
+Use it to end a run or count lost work on the final try: when a retryable
+error (429, 5xx, network) is about to be rethrown and `isLastAttempt` is
+`true`, the message goes to the DLQ next, so mark the run failed or count the
+message's records as failed before rethrowing.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) await this.failRun(data.processId, error);
+        throw error;
+    }
+}
+```
+
+The single source of the max receive count is
+`INTEGRATION_QUEUE_MAX_RECEIVE_COUNT` in `queues/queue-delivery.js`. The
+devtools integration builder uses it for the queue's `RedrivePolicy` and sets
+it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.
@@ -437,22 +523,24 @@ if (!userId) {
 
 ### 9. Logging System (`/logs`)
 
-**Purpose**: Structured logging with debug capabilities.
+**Purpose**: One redacted JSON record per line to stdout (ADR-048). See `docs/guides/LOGGING.md`.
 
-**Functions**:
-- `debug(message, data)` - Debug logging
-- `initDebugLog(eventName, event)` - Initialize debug context
-- `flushDebugLog(error)` - Flush logs on error
+**Rules**:
+- `logs/` is a leaf: it requires only Node built-ins and sibling `logs/` files.
+- Integrations and API modules use `this.logger`; core uses one `getLogger('frigg.<area>')` per module.
+- Fixed message text; ids and counts go into fields. `frigg.*` records at WARN and above need an `eventName` (`<logger name>.<action>`); tests fail without it.
+- Log an error one time, at the boundary (`createHandler`, the express middleware, `Worker.run`, the DLQ processor). Inner code throws with `cause`.
+- Use `statusCode`, not `status` (reserved key). `body`, `payload`, `response` are dropped at INFO and above.
+- Tests assert on `createMemorySink()` records by `eventName`, not on console spies.
+- `debug`, `initDebugLog`, `flushDebugLog` are deprecated shims.
 
 **Usage**:
 
 ```javascript
-const { debug, initDebugLog, flushDebugLog } = require('@friggframework/core');
+const { getLogger } = require('../logs');
+const log = getLogger('frigg.core.sync');
 
-initDebugLog('MyIntegration', event);
-debug('Processing request', { userId, action });
-// ... your code ...
-flushDebugLog(); // On error
+log.warn('Sync skipped', { eventName: 'frigg.core.sync.skipped', processId });
 ```
 
 ### 10. Lambda Utilities (`/lambda`)
@@ -660,8 +748,8 @@ Use test doubles from `@friggframework/test` package for consistent mocking.
 ### Optional
 
 - `SECRET_ARN` - AWS Secrets Manager ARN for auto-injection
-- `DEBUG` - Debug logging pattern
-- `LOG_LEVEL` - Logging level (debug, info, warn, error)
+- `FRIGG_LOG_LEVEL` - Minimum log level (`TRACE` … `FATAL`, default `INFO`; `DEBUG` on local runs)
+- `FRIGG_QUEUE_MAX_RECEIVE_COUNT` - Set by devtools on integration queue workers whose queue the stack owns; feeds `delivery.maxReceiveCount`
 
 ## Version Information
 

package/README.md

@@ -107,10 +107,6 @@ FRIGG_ENCRYPTION_KEY=your-256-bit-encryption-key
 AWS_REGION=us-east-1
 AWS_ACCESS_KEY_ID=your-access-key
 AWS_SECRET_ACCESS_KEY=your-secret-key
-
-# Logging
-DEBUG=frigg:*
-LOG_LEVEL=info
 ```
 
 ## Core Components
@@ -255,10 +251,12 @@ const {
 // Custom business logic error
 throw new RequiredPropertyError('userId is required');
 
-// API communication error
-throw new FetchError('Failed to fetch data from external API', {
-    statusCode: 404,
-    response: errorResponse
+// API communication error. The message is built from the method, the
+// sanitized URL and the status; the response body stays off the message.
+throw await FetchError.create({
+    resource: 'https://api.example.com/contacts',
+    init: { method: 'GET' },
+    response,
 });
 
 // Base error with custom properties
@@ -270,25 +268,25 @@ throw new BaseError('Integration failed', {
 
 ### 6. Logging (`/logs`)
 
-Structured logging with debug capabilities.
+One redacted JSON record per line to stdout, with the correlation ids of
+ADR-011. See the [Logging guide](../../docs/guides/LOGGING.md).
 
 **Usage:**
 ```javascript
-const { debug, initDebugLog, flushDebugLog } = require('@friggframework/core');
-
-// Initialize debug logging
-initDebugLog('integration:slack');
-
-// Log debug information
-debug('Processing webhook payload', { 
-    eventType: 'contact.created',
-    payload: webhookData 
+// In an integration or an API module
+this.logger.info('Contact batch started', {
+    eventName: 'integration.hubspot.batch_started',
+    batchSize: contacts.length,
 });
 
-// Flush logs (useful in serverless environments)
-await flushDebugLog();
+// In core code
+const { getLogger } = require('@friggframework/core');
+const log = getLogger('frigg.core.sync');
+log.warn('Sync skipped', { eventName: 'frigg.core.sync.skipped', processId });
 ```
 
+`debug`, `initDebugLog` and `flushDebugLog` are deprecated shims.
+
 ### 7. User Management (`/user`)
 
 Comprehensive user authentication and authorization system supporting both individual and organizational users.
@@ -1013,8 +1011,7 @@ const {
 | `MONGO_URI` | Yes | MongoDB connection string |
 | `FRIGG_ENCRYPTION_KEY` | Yes | 256-bit encryption key |
 | `AWS_REGION` | No | AWS region for services |
-| `DEBUG` | No | Debug logging pattern |
-| `LOG_LEVEL` | No | Logging level (debug, info, warn, error) |
+| `FRIGG_LOG_LEVEL` | No | Minimum log level: `TRACE`, `DEBUG`, `INFO` (default), `WARN`, `ERROR`, `FATAL`. Local runs default to `DEBUG` |
 
 ## License
 

package/core/CLAUDE.md

@@ -21,13 +21,13 @@ This file provides guidance to Claude Code when working with the Frigg Framework
 - **Database Connection Management**: Automatic MongoDB connection with pooling
 - **Secrets Management**: AWS Secrets Manager integration via `SECRET_ARN` env var
 - **Error Sanitization**: Prevents internal details from leaking to end users
-- **Debug Logging**: Request/response logging with structured debug info
+- **Invocation Scope**: `runInvocationScope` puts `requestId`, `route` and the redacted event summary on every record, then flushes usage, telemetry and sinks
 - **Connection Optimization**: `context.callbackWaitsForEmptyEventLoop = false` for reuse
 
 **Handler Configuration Options**:
 ```javascript
 const handler = createHandler({
-    eventName: 'MyIntegration',           // For logging/debugging
+    eventName: 'MyIntegration',           // Logged as handlerName
     isUserFacingResponse: true,           // true = sanitize errors, false = pass through
     method: async (event, context) => {}, // Your Lambda function logic
     shouldUseDatabase: true               // false = skip MongoDB connection
@@ -52,9 +52,10 @@ const handler = createHandler({
 **Usage Pattern**:
 ```javascript
 class MyWorker extends Worker {
-    async _run(params, context = {}) {
+    async _run(params, context = {}, delivery) {
         // Your job processing logic here
         // params are already JSON.parsed from SQS message body
+        // delivery is { receiveCount, maxReceiveCount, isLastAttempt }
     }
     
     _validateParams(params) {
@@ -129,7 +130,8 @@ class MyIntegration extends Delegate {
 ### Lambda Handler Lifecycle
 1. **Pre-Execution Setup**:
    ```javascript
-   initDebugLog(eventName, event);           // Debug logging setup
+   runInvocationScope({ requestId, handlerName, method, route, invocation }, …); // Logger scope for the invocation
+   log.info('Handler invoked', { eventName: 'frigg.handler.invoked' });
    await secretsToEnv();                     // Secrets Manager injection
    await parametersToEnv();                  // SSM Parameter Store fetch (only when SSM_PARAMETER_PREFIX + FRIGG_SSM_OFFLOADED_KEYS are set)
    context.callbackWaitsForEmptyEventLoop = false; // Connection pooling
@@ -149,12 +151,21 @@ class MyIntegration extends Delegate {
 
 4. **Error Handling & Cleanup**:
    ```javascript
-   flushDebugLog(error);                    // Debug info flush on error
-   // Sanitized error response for user-facing endpoints
+   // One record at the boundary: WARN frigg.handler.rejected (client-safe),
+   // ERROR frigg.handler.failed, or ERROR frigg.handler.halted (halt, no retry).
+   // Sanitized error response for user-facing endpoints; server-to-server
+   // errors are rethrown as a sanitized surrogate (toSanitizedSurrogate).
+   ```
+
+5. **Flush** (in `runInvocationScope`'s `finally`):
+   ```javascript
+   // Telemetry + log sinks under one deadline, fixed at flush start:
+   // min(flushTimeoutMs, remaining time - 50 ms). The usage rollup runs in
+   // parallel with no bound of its own.
    ```
 
 ### SQS Job Processing Lifecycle
-1. **Batch Processing**: Process all records in `event.Records` sequentially
+1. **Batch Processing**: Process all records in `event.Records` sequentially, each inside `runMessageScope` (adds `messageId`, `receiveCount`, `processId`, `integrationId`, `integrationEvent` to its records)
 2. **Message Parsing**: JSON.parse message body for parameters  
 3. **Validation**: Run custom validation on parsed parameters
 4. **Execution**: Call `_run()` method with validated parameters
@@ -205,7 +216,9 @@ const handler = createHandler({
    - Logs full error details internally
 
 2. **Server-to-Server Errors**: `isUserFacingResponse: false`
-   - Re-throws original error for AWS handling
+   - Logs one ERROR, then rethrows a sanitized surrogate for AWS handling:
+     a fresh `Error` with the sanitized `name`, `message`, `stack`, plus
+     `statusCode` and `code`. `instanceof` checks and custom properties are gone
    - Used for SQS, SNS, and internal API calls
    - Enables proper retry mechanisms
 
@@ -214,12 +227,14 @@ const handler = createHandler({
    - Prevents infinite retries for known issues
    - Used for graceful degradation scenarios
 
-### Debug Logging Strategy
+### Logging Strategy
 ```javascript
-initDebugLog(eventName, event);  // Start logging context
-// ... your code ...
-flushDebugLog(error);           // Flush on error (includes full context)
+const { getLogger } = require('../logs');
+const log = getLogger('frigg.core.sync');
+log.info('Sync started', { eventName: 'frigg.core.sync.started', processId });
+// Throw with cause; the boundary logs the error one time.
 ```
+See `docs/guides/LOGGING.md`.
 
 ## Integration Development Patterns
 
@@ -646,7 +661,7 @@ describe('Health Handler', () => {
 const { createHandler } = require('@friggframework/core/core');
 
 const testHandler = createHandler({
-    isUserFacingResponse: false, // Get full errors in tests
+    isUserFacingResponse: false, // Rethrows a sanitized surrogate (name, message, statusCode, code; not the original instance)
     shouldUseDatabase: false,    // Mock/skip DB in tests
     method: yourTestMethod
 });

package/core/Worker.js

@@ -2,8 +2,12 @@ const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
+const { readQueueDelivery } = require('../queues/queue-delivery');
+const { runMessageScope } = require('./invocation-scope');
+const { getLogger } = require('../logs');
 
 const sqs = new SQSClient({ region: process.env.AWS_REGION });
+const log = getLogger('frigg.worker');
 
 class Worker {
     async getQueueURL(params) {
@@ -25,45 +29,36 @@ class Worker {
         );
 
         for (const record of records) {
-            // Log record entry with SQS-provided attributes useful for tracing
-            // delivery history (ApproximateReceiveCount for retries, etc.).
-            let parsedEvent;
-            try {
-                parsedEvent = JSON.parse(record.body)?.event;
-            } catch {
-                parsedEvent = undefined;
-            }
-            console.log(`[Worker] record begin`, {
-                messageId: record.messageId,
-                event: parsedEvent,
-                receiveCount: record.attributes?.ApproximateReceiveCount,
-            });
-
-            try {
-                const runParams = JSON.parse(record.body);
-                this._validateParams(runParams);
-                await this._run(runParams, context);
-                console.log(`[Worker] record success`, {
-                    messageId: record.messageId,
-                    event: runParams?.event,
-                });
-            } catch (error) {
-                if (error.isHaltError) {
-                    // HaltError means "discard this message, don't retry".
-                    // Treat as success so SQS deletes it from the queue.
-                    // Logged explicitly — silent discards made prod debugging
-                    // extremely hard; keep this visible.
-                    console.warn(`[Worker] record halted (discarded, no retry)`, {
-                        messageId: record.messageId,
-                        event: parsedEvent,
-                        reason: error.message,
-                        statusCode: error.statusCode,
+            await runMessageScope(record, async () => {
+                // messageId, receiveCount and the event come from the scope.
+                log.debug('Record started', { eventName: 'frigg.worker.record_started' });
+
+                try {
+                    const runParams = JSON.parse(record.body);
+                    this._validateParams(runParams);
+                    await this._run(runParams, context, readQueueDelivery(record));
+                    log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
+                } catch (error) {
+                    if (error.isHaltError) {
+                        // HaltError means "discard this message, don't retry".
+                        // Treat as success so SQS deletes it from the queue.
+                        // Logged explicitly — silent discards made prod debugging
+                        // extremely hard; keep this visible.
+                        log.error('Record halted (discarded, no retry)', {
+                            eventName: 'frigg.worker.record_halted',
+                            statusCode: error.statusCode,
+                            error,
+                        });
+                        return;
+                    }
+                    // The message goes back to SQS, so WARN (ADR-048 §4).
+                    log.warn('Record failed, returned for retry', {
+                        eventName: 'frigg.worker.record_failed',
+                        error,
                     });
-                    continue;
+                    batchItemFailures.push({ itemIdentifier: record.messageId });
                 }
-                console.error(`[Worker] Failed to process record ${record.messageId}:`, error);
-                batchItemFailures.push({ itemIdentifier: record.messageId });
-            }
+            });
         }
 
         if (batchItemFailures.length > 0) {