@empire-builder-kit/nx 1.0.0-rc.4 → 1.0.0-rc.41

package/src/generators/slice/built-in-plan-sql-events.js

@@ -0,0 +1,276 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderSqlEventFiles = renderSqlEventFiles;
+/** SQL is additive: its transactions/outbox never replace the native data store. */
+function renderSqlEventFiles(name) {
+    const pascalName = name.replace(/(^|-)([a-z])/g, (_match, _dash, letter) => letter.toUpperCase());
+    return {
+        'database/sql-events.ts': `import { createHash } from 'node:crypto';
+import { EventBridgeClient, PutEventsCommand, type PutEventsRequestEntry } from '@aws-sdk/client-eventbridge';
+import { buildEventEntry, parseEventEnvelope, type EventContractRef, type BuildEventEntryOptions } from '@empire-builder-kit/runtime/events';
+import type { Client } from 'pg';
+import backgroundWork from '../operations/background-work.json' with { type: 'json' };
+import { withPrimaryDatabase } from './runtime-client';
+
+type Context = { correlationId: string };
+type Workload = Parameters<typeof withPrimaryDatabase>[1];
+type Emit = <Data>(contract: EventContractRef, data: Data, options: Pick<BuildEventEntryOptions, 'eventId' | 'occurredAt' | 'actor'>) => Promise<void>;
+interface Scope { stage: string; busName: string; enabled: boolean }
+interface Dependencies {
+  scope(): Scope;
+  withDatabase: typeof withPrimaryDatabase;
+  publish(entry: PutEventsRequestEntry): Promise<void>;
+}
+
+function validateScope(scope: Scope) {
+  if (!scope || !/^(development|staging|production|personal-[a-f0-9]{24})$/.test(scope.stage) ||
+      typeof scope.busName !== 'string' || !scope.busName.trim() || scope.busName.trim() !== scope.busName ||
+      typeof scope.enabled !== 'boolean') throw new Error('SQL outbox requires its exact stage, bus and maintenance state.');
+}
+function validateEntry(entry: PutEventsRequestEntry, scope: Scope) {
+  if (!entry || entry.Source !== '${name}' || entry.EventBusName !== scope.busName ||
+      typeof entry.DetailType !== 'string' || !entry.DetailType || typeof entry.Detail !== 'string' ||
+      Buffer.byteLength(entry.Detail) > 256 * 1024) throw new Error('SQL event differs from its Slice/bus scope.');
+  const envelope = parseEventEnvelope(JSON.parse(entry.Detail));
+  if (envelope.meta.stage !== scope.stage || envelope.meta.slice !== '${name}')
+    throw new Error('SQL event differs from its exact stage.');
+  return envelope;
+}
+
+/** Own the SQL transaction; emit must be awaited inside operation. No cross-store atomicity is implied. */
+export function createSqlEventOutbox(dependencies: Dependencies) {
+  async function notify(scope: Scope, context: Context) {
+    if (!scope.enabled) return 'paused' as const;
+    try {
+      await dependencies.publish({ Source: 'ebk.sql-outbox.${name}', DetailType: 'SqlOutboxReady',
+        EventBusName: scope.busName, Detail: JSON.stringify({ schemaVersion: 1, slice: '${name}', stage: scope.stage, correlationId: context.correlationId }) });
+      return 'notified' as const;
+    } catch {
+      // Domain work already committed. Do not retry it because a wake-up was lost.
+      console.warn(JSON.stringify({ kind: 'ebk-sql-outbox-notification', outcome: 'reconciliation-required', slice: '${name}', stage: scope.stage }));
+      return 'reconciliation-required' as const;
+    }
+  }
+  async function transaction<Result>(context: Context, workload: Workload,
+    operation: (client: Client, emit: Emit) => Promise<Result>) {
+    const scope = dependencies.scope(); validateScope(scope);
+    let queued = 0;
+    const value = await dependencies.withDatabase(context, workload, async (client) => {
+      await client.query('BEGIN');
+      let open = true, invalid = false;
+      const emissions: Promise<void>[] = [];
+      const append: Emit = async (contract, data, options) => {
+        try {
+          if (!open || ++queued > 100) throw new Error('SQL event transaction is closed or exceeds 100 intents.');
+          const built = buildEventEntry(contract, data, { ...options, context, stage: scope.stage, busName: scope.busName });
+          const entry = { Source: built.entry.source, DetailType: built.entry.detailType, Detail: built.entry.detail, EventBusName: built.entry.eventBusName };
+          const envelope = validateEntry(entry, scope);
+          const id = createHash('sha256').update(scope.stage + '/${name}/' + envelope.meta.eventId).digest('hex');
+          const json = JSON.stringify(entry);
+          await client.query(\`INSERT INTO data_lifecycle_outbox (idempotency_key, event_json, expires_at)
+            VALUES ($1, $2::jsonb, now() + interval '30 days') ON CONFLICT (idempotency_key) DO NOTHING\`, [id, json]);
+          const existing = await client.query<{ matches: boolean }>(\`SELECT event_json = $2::jsonb AS matches
+            FROM data_lifecycle_outbox WHERE idempotency_key = $1 FOR UPDATE\`, [id, json]);
+          if (existing.rows.length !== 1 || existing.rows[0].matches !== true)
+            throw new Error('SQL event ID was reused with different content.');
+        } catch (error) { invalid = true; throw error; }
+      };
+      const emit: Emit = (...args) => {
+        const pending = append(...args);
+        emissions.push(pending);
+        void pending.catch(() => undefined);
+        return pending;
+      };
+      try {
+        const result = await operation(client, emit);
+        open = false;
+        await Promise.all(emissions);
+        if (invalid) throw new Error('SQL transaction contains a rejected event intent.');
+        await client.query('COMMIT');
+        return result;
+      } catch (error) {
+        open = false;
+        await Promise.allSettled(emissions);
+        await client.query('ROLLBACK').catch(() => undefined);
+        throw error;
+      }
+    });
+    return { value, queued, notification: queued ? await notify(scope, context) : 'not-needed' as const };
+  }
+  async function reconcile(context: Context) {
+    const scope = dependencies.scope(); validateScope(scope);
+    if (!scope.enabled) return { outcome: 'paused', published: 0, failed: 0, remaining: null, maxAttempts: null };
+    const result = await dependencies.withDatabase(context, 'async-outbox-relay', async (client) => {
+      await client.query('BEGIN');
+      try {
+        // Failed rows gain an attempt so one poison event cannot starve later work.
+        // Pending intents do not expire merely because their wake-up was missed.
+        const pending = await client.query<{ id: string; entry: PutEventsRequestEntry }>(\`SELECT idempotency_key AS id, event_json AS entry
+          FROM data_lifecycle_outbox WHERE published_at IS NULL
+          ORDER BY attempts, created_at, idempotency_key LIMIT 10 FOR UPDATE SKIP LOCKED\`);
+        let published = 0, failed = 0;
+        for (const row of pending.rows) {
+          try {
+            validateEntry(row.entry, scope);
+            await dependencies.publish(row.entry);
+          } catch {
+            failed++;
+            await client.query(\`UPDATE data_lifecycle_outbox SET attempts = attempts + 1,
+              last_error_code = 'event-publication-failed' WHERE idempotency_key = $1\`, [row.id]);
+            continue;
+          }
+          // A publish/commit interruption may redeliver the same event ID.
+          await client.query(\`UPDATE data_lifecycle_outbox SET attempts = attempts + 1, last_error_code = NULL,
+            published_at = now(), expires_at = now() + interval '30 days' WHERE idempotency_key = $1 AND published_at IS NULL\`, [row.id]);
+          published++;
+        }
+        const count = await client.query<{ count: string; max_attempts: string }>(\`SELECT count(*)::text AS count,
+          coalesce(max(attempts), 0)::text AS max_attempts FROM data_lifecycle_outbox WHERE published_at IS NULL\`);
+        const remaining = Number(count.rows[0]?.count);
+        if (!Number.isSafeInteger(remaining) || remaining < 0) throw new Error('SQL outbox backlog is invalid.');
+        // Reporting attempts is telemetry only; it never expires or drops an intent.
+        const attempts = Number(count.rows[0]?.max_attempts);
+        const maxAttempts = Number.isSafeInteger(attempts) && attempts >= 0 ? attempts : null;
+        // Bounded control-metadata cleanup; pending intents never expire. Use
+        // row locks so overlapping relays cannot delete another worker's batch.
+        const pruned = await client.query(\`DELETE FROM data_lifecycle_outbox WHERE idempotency_key IN (
+          SELECT idempotency_key FROM data_lifecycle_outbox
+          WHERE published_at IS NOT NULL AND expires_at <= now()
+          ORDER BY expires_at, idempotency_key LIMIT 100 FOR UPDATE SKIP LOCKED
+        )\`);
+        await client.query('COMMIT');
+        return { outcome: failed ? 'retry-required' : remaining ? 'pending' : 'converged', published, failed, remaining, maxAttempts, pruned: pruned.rowCount ?? 0 };
+      } catch (error) { await client.query('ROLLBACK').catch(() => undefined); throw error; }
+    });
+    // Continue a healthy backlog promptly; failures remain retryable and recover hourly.
+    if (result.remaining > 0 && result.failed === 0) await notify(scope, context);
+    return result;
+  }
+  return { transaction, reconcile };
+}
+
+const eventBridge = new EventBridgeClient({ maxAttempts: 2 });
+export const sqlEventOutbox = createSqlEventOutbox({
+  scope: () => {
+    const stage = process.env.EBK_STAGE ?? '';
+    const policyStage = /^personal-[a-f0-9]{24}$/.test(stage) ? 'development' : stage;
+    const stages = backgroundWork.stages as Record<string, { sqlOutboxRecovery?: { enabled?: boolean } }>;
+    return { stage, busName: process.env.EVENT_BUS_NAME ?? '', enabled: stages[policyStage]?.sqlOutboxRecovery?.enabled as boolean };
+  },
+  withDatabase: withPrimaryDatabase,
+  publish: async (entry) => {
+    const result = await eventBridge.send(new PutEventsCommand({ Entries: [entry] }));
+    if (result.FailedEntryCount !== 0 || result.Entries?.length !== 1 || !result.Entries[0].EventId || result.Entries[0].ErrorCode)
+      throw new Error('EventBridge did not acknowledge the SQL event.');
+  },
+});
+`,
+        'async/sql-outbox-relay.ts': `import { randomUUID } from 'node:crypto';
+import { createMetricEmitter } from '@empire-builder-kit/runtime/observability';
+import { sqlEventOutbox } from '../database/sql-events.js';
+
+const metrics = createMetricEmitter();
+
+export async function handler(event: { id?: string } = {}) {
+  const started = Date.now();
+  const result = await sqlEventOutbox.reconcile({ correlationId: event.id ?? randomUUID() });
+  console.info(JSON.stringify({ kind: 'ebk-sql-outbox-relay', ...result, durationMilliseconds: Date.now() - started }));
+  // Observational gauges precede the failure signal. A paused relay reads
+  // nothing and reports no backlog; a lost metric never changes delivery.
+  for (const [metricName, value] of [['SqlOutboxBacklog', result.remaining], ['SqlOutboxMaxAttempts', result.maxAttempts]] as const) {
+    if (value !== null) metrics.emit({ namespace: 'EBK/Async', metricName, unit: 'Count', value,
+      dimensions: { slice: '${name}', stage: process.env.EBK_STAGE ?? 'local' } });
+  }
+  if (result.failed) throw new Error('SQL outbox retains failed deliveries; inspect content-free diagnostics and retry.');
+  return result;
+}
+`,
+        'infra/sql-outbox.ts': `export function createSqlOutboxInfrastructure(input: {
+  workspace: string; database: any; secretLinks: any[]; eventBus: any; eventBusName: any; vpc: any;
+  schedule: { scheduleExpression: string; state: 'ENABLED' | 'DISABLED' };
+  alertTopicArn: any;
+  managedLogGroup(logicalName: string, component: string, alertTopicArn: any): { logGroup: { name: any } };
+  lambdaLogging(logGroupName: any): { logging: any; transform: any };
+}) {
+  const tags = { 'ebk:slice': '${name}', 'ebk:stage': $app.stage };
+  // Feature-owned SQL intents have a separate relay; the framework core stays DynamoDB.
+  const relayLogs = input.managedLogGroup('${pascalName}SqlOutboxRelayLogs', 'sql-outbox-relay', input.alertTopicArn);
+  const relay = new sst.aws.Function('${pascalName}SqlOutboxRelay', {
+    handler: 'async/sql-outbox-relay.handler',
+    environment: { EBK_STAGE: $app.stage, EBK_WORKSPACE: input.workspace,
+      EBK_WORKLOAD_IDENTITY: '${name}:async-outbox-relay', EVENT_BUS_NAME: input.eventBusName },
+    link: [input.database, input.eventBus, ...input.secretLinks], vpc: input.vpc,
+    // The SQL transaction's FOR UPDATE SKIP LOCKED owns concurrent delivery.
+    // Do not require an account-level reserved-concurrency quota for this relay.
+    timeout: '1 minute', ...input.lambdaLogging(relayLogs.logGroup.name),
+  });
+  const account = aws.getCallerIdentityOutput().accountId;
+  const notification = new aws.cloudwatch.EventRule('${pascalName}SqlOutboxNotification', {
+    eventBusName: input.eventBusName, state: input.schedule.state, tags,
+    eventPattern: $jsonStringify({ account: [account], source: ['ebk.sql-outbox.${name}'],
+      'detail-type': ['SqlOutboxReady'], detail: { schemaVersion: [1], slice: ['${name}'], stage: [$app.stage] } }),
+  });
+  const notificationPermission = new aws.lambda.Permission('${pascalName}SqlOutboxNotificationPermission', {
+    action: 'lambda:InvokeFunction', function: relay.arn, principal: 'events.amazonaws.com',
+    sourceArn: notification.arn, sourceAccount: account,
+  });
+  const target = new aws.cloudwatch.EventTarget('${pascalName}SqlOutboxNotificationTarget', {
+    arn: relay.arn, eventBusName: input.eventBusName, rule: notification.name,
+    retryPolicy: { maximumEventAgeInSeconds: 3600, maximumRetryAttempts: 2 },
+  }, { dependsOn: [notificationPermission] });
+    const rule = new aws.cloudwatch.EventRule('${pascalName}SqlOutboxRecoverySchedule', {
+      ...input.schedule, tags, description: 'Recovery of missed SQL outbox notifications.',
+    });
+    const permission = new aws.lambda.Permission('${pascalName}SqlOutboxRecoveryPermission', {
+      action: 'lambda:InvokeFunction', function: relay.arn, principal: 'events.amazonaws.com', sourceArn: rule.arn, sourceAccount: account,
+    });
+    const recovery = new aws.cloudwatch.EventTarget('${pascalName}SqlOutboxRecoveryTarget', {
+      arn: relay.arn, rule: rule.name, retryPolicy: { maximumEventAgeInSeconds: 3600, maximumRetryAttempts: 2 },
+    }, { dependsOn: [permission] });
+  return { relay, notification, target, recovery };
+}
+`,
+        'docs/runbooks/sql-events.md': `# Relational transactions and event delivery
+
+Declare SQL only for a relational requirement. Existing DynamoDB records, inboxes
+and event intents stay in DynamoDB; there is no transaction spanning both stores.
+Use the generated database/sql-events.ts helper from a seeded feature repository:
+
+\`\`\`ts
+const result = await sqlEventOutbox.transaction(context, 'api-database-records', async (client, emit) => {
+  // Authorize the user before entering; write your declared relational schema here.
+  await client.query('INSERT INTO invoice_ledger (id, amount) VALUES ($1, $2)', [id, amount]);
+  await emit(InvoiceRecorded, { id, amount }, { eventId, occurredAt });
+  return { id };
+});
+\`\`\`
+
+The helper owns BEGIN/COMMIT/ROLLBACK. Await every emit inside the callback; do
+not commit or detach work yourself. Domain changes and event intents commit
+together. A duplicate event ID with different content rolls the transaction back.
+Use stable event IDs and caller idempotency for repeat requests.
+
+Only after commit does a content-free notification wake the SQL relay. A missed
+notification does not undo committed data: the result reports reconciliation-
+required, and the retained outbox is recovered on the configured schedule.
+Pending intents are not discarded on age alone. A relay run that retains a
+failed delivery fails its invocation (\`sql-outbox-relay-errors\`). Each enabled
+run reports the global unpublished count as \`SqlOutboxBacklog\` and the highest
+unpublished attempt count as \`SqlOutboxMaxAttempts\`; \`sql-outbox-backlog\`
+alarms when every run in two consecutive hours ended with a backlog and
+\`sql-outbox-attempts\` at ten attempts. These are observations, never delivery
+rules. See docs/runbooks/async-recovery.md. Published intents retain thirty
+days of replay metadata. Each enabled SQL relay invocation prunes at most one
+hundred expired published intents in the same transaction. Pausing SQL recovery
+also defers this cleanup. Pending intents are never pruned. Delivery is at least
+once; consumers must deduplicate.
+
+SQL pause/cadence use sqlOutboxRecovery in operations/background-work.json, separate from
+Aurora capacity. Pausing blocks both notifications and relay reads. Resume
+deliberately and review backlog; no maintenance state is changed by this helper.
+The feature's runtime must link its declared SQL database and Foundation event
+bus through supported generation, with their existing exact runtime permissions.
+`,
+    };
+}
+//# sourceMappingURL=built-in-plan-sql-events.js.map

package/src/generators/slice/built-in-plan-testing-files.d.ts

@@ -1,2 +1,2 @@
 /** @internal */
-export declare function renderProductTestingFiles(name: string, pascalName: string): Readonly<Record<string, string>>;
+export declare function renderProductTestingFiles(name: string, pascalName: string, sql?: boolean): Readonly<Record<string, string>>;