mcp-event-intelligence 0.1.1 → 0.2.0

package/scripts/lib/composite-wake-coordinator.mjs

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import {
   WakeRecordSchema,
   sha256Hex,
@@ -11,6 +12,13 @@ async function stableId(prefix, value) {
   return `${prefix}_${hash.slice(0, 24)}`;
 }
 
+function retryDelayMs(attemptCount, baseDelayMs, maxDelayMs) {
+  return Math.min(
+    maxDelayMs,
+    baseDelayMs * (2 ** Math.max(0, attemptCount - 1)),
+  );
+}
+
 export class CompositeWakeCoordinator {
   constructor({
     store,
@@ -18,12 +26,91 @@ export class CompositeWakeCoordinator {
     deliverer = null,
     packetBuilder = buildGenericRuntimeWakePacket,
     now = () => new Date(),
+    workerId = `wake-worker:${randomUUID()}`,
+    leaseMs = 30000,
+    maxAttempts = 5,
+    retryBaseDelayMs = 1000,
+    retryMaxDelayMs = 60000,
   }) {
     this.store = store;
     this.triggerEngine = triggerEngine;
     this.deliverer = deliverer;
     this.packetBuilder = packetBuilder;
     this.now = now;
+    this.workerId = workerId;
+    this.leaseMs = Math.max(1000, Number(leaseMs) || 30000);
+    this.maxAttempts = Math.max(1, Number(maxAttempts) || 5);
+    this.retryBaseDelayMs = Math.max(1, Number(retryBaseDelayMs) || 1000);
+    this.retryMaxDelayMs = Math.max(
+      this.retryBaseDelayMs,
+      Number(retryMaxDelayMs) || 60000,
+    );
+
+    for (const method of [
+      'ensureWakeDelivery',
+      'getWakeDelivery',
+      'claimWakeDelivery',
+      'completeWakeDelivery',
+      'failWakeDelivery',
+    ]) {
+      if (typeof store?.[method] !== 'function') {
+        const error = new Error(
+          `Wake coordinator store requires ${method}() for durable delivery`,
+        );
+        error.code = 'WAKE_DELIVERY_STORE_UNSUPPORTED';
+        throw error;
+      }
+    }
+  }
+
+  async deliveredResult({ match, wakeId, definition, delivery }) {
+    const existing = this.store.latestWake(wakeId);
+    let delivered = existing;
+    if (
+      !delivered ||
+      (delivered.status !== 'delivered' && delivered.status !== 'handled')
+    ) {
+      delivered = WakeRecordSchema.parse({
+        wakeId,
+        traceId: match.sourceEvents[0]?.traceId ?? match.matchId,
+        decisionId: null,
+        subscriptionId: `trigger:${match.triggerId}`,
+        sourceEventId: match.matchId,
+        createdAt: this.now().toISOString(),
+        target: definition.target,
+        status: 'delivered',
+        runtimeReceiptId: delivery.runtimeReceiptId,
+      });
+      await this.store.appendWake(delivered);
+      await this.store.appendAudit({
+        auditId: await stableId(
+          'audit',
+          `${wakeId}:delivered-recovered:${this.store.auditLength()}`,
+        ),
+        traceId: delivered.traceId,
+        timestamp: this.now().toISOString(),
+        kind: 'wake.delivered',
+        entityType: 'wake',
+        entityId: wakeId,
+        fromState: 'wake_queued',
+        toState: 'wake_delivered',
+        details: {
+          triggerMatchId: match.matchId,
+          runtimeReceiptId: delivered.runtimeReceiptId ?? null,
+          recoveredFromDeliveryState: true,
+          composite: true,
+        },
+      });
+    }
+
+    if (match.status !== 'fired') {
+      await this.triggerEngine.markFired(match.matchId, wakeId);
+    }
+
+    return {
+      status: delivered.status === 'handled' ? 'handled' : 'wake_delivered',
+      wake: delivered,
+    };
   }
 
   async deliverMatched(match) {
@@ -61,16 +148,19 @@ export class CompositeWakeCoordinator {
       `composite:${match.matchId}:${definition.target.runtime}:${definition.target.id}`,
     );
 
-    const existing = this.store.latestWake(wakeId);
-    if (existing?.status === 'handled' || existing?.status === 'delivered') {
+    const existingWake = this.store.latestWake(wakeId);
+    if (existingWake?.status === 'handled' || existingWake?.status === 'delivered') {
       if (match.status !== 'fired') {
         await this.triggerEngine.markFired(match.matchId, wakeId);
       }
       return {
-        status: existing.status === 'handled' ? 'handled' : 'wake_delivered',
-        wake: existing,
+        status: existingWake.status === 'handled' ? 'handled' : 'wake_delivered',
+        wake: existingWake,
       };
     }
+    if (existingWake?.status === 'dead_letter') {
+      return { status: 'dead_letter', wake: existingWake };
+    }
 
     const queued = WakeRecordSchema.parse({
       wakeId,
@@ -83,8 +173,88 @@ export class CompositeWakeCoordinator {
       status: 'queued',
     });
 
-    if (!existing || existing.status === 'dead_letter') {
+    if (!this.deliverer) {
+      if (!existingWake) {
+        await this.store.appendWake(queued);
+        await this.store.appendAudit({
+          auditId: await stableId(
+            'audit',
+            `${wakeId}:queued:${this.store.auditLength()}`,
+          ),
+          traceId: queued.traceId,
+          timestamp: this.now().toISOString(),
+          kind: 'wake.queued',
+          entityType: 'wake',
+          entityId: wakeId,
+          toState: 'wake_queued',
+          details: {
+            triggerMatchId: match.matchId,
+            triggerId: match.triggerId,
+            composite: true,
+          },
+        });
+      }
+      return {
+        status: 'wake_queued',
+        wake: this.store.latestWake(wakeId) ?? queued,
+      };
+    }
+
+    const nowIso = this.now().toISOString();
+    await this.store.ensureWakeDelivery({
+      wakeId,
+      matchId: match.matchId,
+      triggerId: match.triggerId,
+      triggerVersion: match.triggerVersion,
+      runtime: definition.target.runtime,
+      now: nowIso,
+    });
+
+    const currentDelivery = this.store.getWakeDelivery(wakeId);
+    if (currentDelivery?.status === 'delivered') {
+      return this.deliveredResult({
+        match,
+        wakeId,
+        definition,
+        delivery: currentDelivery,
+      });
+    }
+    if (currentDelivery?.status === 'dead_letter') {
+      const deadLetter = existingWake?.status === 'dead_letter'
+        ? existingWake
+        : WakeRecordSchema.parse({
+            ...queued,
+            createdAt: this.now().toISOString(),
+            status: 'dead_letter',
+          });
+      if (existingWake?.status !== 'dead_letter') {
+        await this.store.appendWake(deadLetter);
+      }
+      return { status: 'dead_letter', wake: deadLetter };
+    }
+
+    const claim = await this.store.claimWakeDelivery(wakeId, {
+      workerId: this.workerId,
+      now: nowIso,
+      leaseMs: this.leaseMs,
+    });
+
+    if (!claim) {
+      const delivery = this.store.getWakeDelivery(wakeId);
+      return {
+        status:
+          delivery?.status === 'claimed'
+            ? 'delivery_in_progress'
+            : 'retry_scheduled',
+        wake: this.store.latestWake(wakeId) ?? queued,
+        delivery,
+      };
+    }
+
+    let persistedQueued = this.store.latestWake(wakeId);
+    if (!persistedQueued) {
       await this.store.appendWake(queued);
+      persistedQueued = queued;
       await this.store.appendAudit({
         auditId: await stableId(
           'audit',
@@ -104,10 +274,6 @@ export class CompositeWakeCoordinator {
       });
     }
 
-    if (!this.deliverer) {
-      return { status: 'wake_queued', wake: queued };
-    }
-
     try {
       const packet = this.packetBuilder({
         wakeId,
@@ -115,6 +281,19 @@ export class CompositeWakeCoordinator {
         definition,
       });
       const receipt = await this.deliverer(packet);
+      if (
+        !receipt ||
+        typeof receipt.runtimeReceiptId !== 'string' ||
+        !receipt.runtimeReceiptId
+      ) {
+        throw new Error('Wake deliverer must return runtimeReceiptId');
+      }
+
+      const delivery = await this.store.completeWakeDelivery(wakeId, {
+        workerId: this.workerId,
+        runtimeReceiptId: receipt.runtimeReceiptId,
+        now: this.now().toISOString(),
+      });
 
       const delivered = WakeRecordSchema.parse({
         ...queued,
@@ -139,6 +318,7 @@ export class CompositeWakeCoordinator {
           triggerMatchId: match.matchId,
           runtimeReceiptId: delivered.runtimeReceiptId,
           duplicate: receipt.duplicate === true,
+          attemptCount: delivery?.attemptCount ?? claim.attemptCount,
           composite: true,
         },
       });
@@ -149,8 +329,50 @@ export class CompositeWakeCoordinator {
         status: 'wake_delivered',
         wake: delivered,
         packet,
+        delivery,
       };
     } catch (error) {
+      const delay = retryDelayMs(
+        claim.attemptCount,
+        this.retryBaseDelayMs,
+        this.retryMaxDelayMs,
+      );
+      const failureAt = this.now();
+      const delivery = await this.store.failWakeDelivery(wakeId, {
+        workerId: this.workerId,
+        error,
+        now: failureAt.toISOString(),
+        maxAttempts: this.maxAttempts,
+        nextAttemptAt: new Date(failureAt.getTime() + delay).toISOString(),
+      });
+
+      if (delivery?.status !== 'dead_letter') {
+        await this.store.appendAudit({
+          auditId: await stableId(
+            'audit',
+            `${wakeId}:retry:${delivery?.attemptCount ?? claim.attemptCount}:${this.store.auditLength()}`,
+          ),
+          traceId: queued.traceId,
+          timestamp: this.now().toISOString(),
+          kind: 'wake.retry_scheduled',
+          entityType: 'wake',
+          entityId: wakeId,
+          details: {
+            triggerMatchId: match.matchId,
+            attemptCount: delivery?.attemptCount ?? claim.attemptCount,
+            nextAttemptAt: delivery?.nextAttemptAt ?? null,
+            message: error instanceof Error ? error.message : 'wake delivery failed',
+            composite: true,
+          },
+        });
+        return {
+          status: 'retry_scheduled',
+          wake: persistedQueued,
+          delivery,
+          error,
+        };
+      }
+
       const deadLetter = WakeRecordSchema.parse({
         ...queued,
         createdAt: this.now().toISOString(),
@@ -171,6 +393,7 @@ export class CompositeWakeCoordinator {
         toState: 'dead_letter',
         details: {
           triggerMatchId: match.matchId,
+          attemptCount: delivery?.attemptCount ?? claim.attemptCount,
           message: error instanceof Error ? error.message : 'wake delivery failed',
           composite: true,
         },
@@ -178,6 +401,7 @@ export class CompositeWakeCoordinator {
       return {
         status: 'dead_letter',
         wake: deadLetter,
+        delivery,
         error,
       };
     }

package/scripts/lib/event-processor.mjs

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import {
   assertDecisionLineage,
   assertWakeLineage,
@@ -20,17 +21,37 @@ async function stableId(prefix, value) {
   return `${prefix}_${hash.slice(0, 24)}`;
 }
 
+function retryDelayMs(attemptCount, baseDelayMs, maxDelayMs) {
+  return Math.min(
+    maxDelayMs,
+    baseDelayMs * (2 ** Math.max(0, attemptCount - 1)),
+  );
+}
+
 export class EventProcessor {
   constructor({
     store,
     evaluator = null,
     wakeDeliverer = null,
     now = () => new Date(),
+    workerId = `event-wake-worker:${randomUUID()}`,
+    leaseMs = 30000,
+    maxAttempts = 5,
+    retryBaseDelayMs = 1000,
+    retryMaxDelayMs = 60000,
   }) {
     this.store = store;
     this.evaluator = evaluator;
     this.wakeDeliverer = wakeDeliverer;
     this.now = now;
+    this.workerId = workerId;
+    this.leaseMs = Math.max(1000, Number(leaseMs) || 30000);
+    this.maxAttempts = Math.max(1, Number(maxAttempts) || 5);
+    this.retryBaseDelayMs = Math.max(1, Number(retryBaseDelayMs) || 1000);
+    this.retryMaxDelayMs = Math.max(
+      this.retryBaseDelayMs,
+      Number(retryMaxDelayMs) || 60000,
+    );
   }
 
   async ingest(rawRequest) {
@@ -307,73 +328,267 @@ export class EventProcessor {
       };
     }
 
+    const deliveryResult = await this.deliverQueuedWake({
+      wake: queuedWake,
+      lineage,
+      event: request.event,
+      decision,
+    });
+
+    return {
+      traceId,
+      sourceEventId: request.event.eventId,
+      status: deliveryResult.status,
+      decision,
+      wake: deliveryResult.wake,
+    };
+  }
+
+  async deliveredWakeFromState(wake, delivery) {
+    const latest = this.store.latestWake(wake.wakeId);
+    if (latest?.status === 'handled' || latest?.status === 'delivered') {
+      return latest;
+    }
+
+    const delivered = WakeRecordSchema.parse({
+      ...wake,
+      createdAt: nowIso(this.now),
+      status: 'delivered',
+      runtimeReceiptId: delivery.runtimeReceiptId ?? undefined,
+    });
+    await this.store.appendWake(delivered);
+    await this.store.appendAudit({
+      auditId: await stableId(
+        'audit',
+        `${wake.traceId}:wake-delivered-recovered:${this.store.auditLength()}`,
+      ),
+      traceId: wake.traceId,
+      timestamp: nowIso(this.now),
+      kind: 'wake.delivered',
+      entityType: 'wake',
+      entityId: wake.wakeId,
+      fromState: 'wake_queued',
+      toState: 'wake_delivered',
+      details: {
+        runtimeReceiptId: delivered.runtimeReceiptId ?? null,
+        recoveredFromDeliveryState: true,
+      },
+    });
+    return delivered;
+  }
+
+  async deliverQueuedWake({ wake, lineage, event, decision }) {
+    if (!this.wakeDeliverer) {
+      return { status: 'wake_queued', wake };
+    }
+
+    const now = nowIso(this.now);
+    await this.store.ensureWakeDelivery({
+      wakeId: wake.wakeId,
+      sourceType: 'event',
+      sourceId: wake.traceId,
+      runtime: wake.target.runtime,
+      now,
+    });
+
+    const current = this.store.getWakeDelivery(wake.wakeId);
+    if (current?.status === 'delivered') {
+      return {
+        status: 'wake_delivered',
+        wake: await this.deliveredWakeFromState(wake, current),
+      };
+    }
+    if (current?.status === 'dead_letter') {
+      return {
+        status: 'dead_letter',
+        wake: this.store.latestWake(wake.wakeId) ?? wake,
+      };
+    }
+
+    const claim = await this.store.claimWakeDelivery(wake.wakeId, {
+      workerId: this.workerId,
+      now,
+      leaseMs: this.leaseMs,
+    });
+    if (!claim) {
+      const delivery = this.store.getWakeDelivery(wake.wakeId);
+      return {
+        status:
+          delivery?.status === 'claimed'
+            ? 'delivery_in_progress'
+            : 'retry_scheduled',
+        wake: this.store.latestWake(wake.wakeId) ?? wake,
+      };
+    }
+
     try {
       const receipt = await this.wakeDeliverer({
-        wake: queuedWake,
+        wake,
         lineage,
-        event: request.event,
+        event,
         decision,
       });
+      if (
+        !receipt ||
+        typeof receipt.runtimeReceiptId !== 'string' ||
+        !receipt.runtimeReceiptId
+      ) {
+        throw new Error('Wake deliverer must return runtimeReceiptId');
+      }
 
+      const delivery = await this.store.completeWakeDelivery(wake.wakeId, {
+        workerId: this.workerId,
+        runtimeReceiptId: receipt.runtimeReceiptId,
+        now: nowIso(this.now),
+      });
       const deliveredWake = WakeRecordSchema.parse({
-        ...queuedWake,
+        ...wake,
         createdAt: nowIso(this.now),
         status: 'delivered',
-        runtimeReceiptId: receipt?.runtimeReceiptId ?? undefined,
+        runtimeReceiptId: receipt.runtimeReceiptId,
       });
       await this.store.appendWake(deliveredWake);
       await this.store.appendAudit({
-        auditId: await stableId('audit', `${traceId}:wake-delivered`),
-        traceId,
+        auditId: await stableId(
+          'audit',
+          `${wake.traceId}:wake-delivered:${this.store.auditLength()}`,
+        ),
+        traceId: wake.traceId,
         timestamp: nowIso(this.now),
         kind: 'wake.delivered',
         entityType: 'wake',
-        entityId: wakeId,
+        entityId: wake.wakeId,
         fromState: 'wake_queued',
         toState: 'wake_delivered',
         details: {
           runtimeReceiptId: deliveredWake.runtimeReceiptId ?? null,
+          attemptCount: delivery?.attemptCount ?? claim.attemptCount,
         },
       });
-
       return {
-        traceId,
-        sourceEventId: request.event.eventId,
         status: 'wake_delivered',
-        decision,
         wake: deliveredWake,
       };
     } catch (error) {
+      const failedAt = this.now();
+      const delay = retryDelayMs(
+        claim.attemptCount,
+        this.retryBaseDelayMs,
+        this.retryMaxDelayMs,
+      );
+      const delivery = await this.store.failWakeDelivery(wake.wakeId, {
+        workerId: this.workerId,
+        error,
+        now: failedAt.toISOString(),
+        maxAttempts: this.maxAttempts,
+        nextAttemptAt: new Date(failedAt.getTime() + delay).toISOString(),
+      });
+
+      if (delivery?.status !== 'dead_letter') {
+        await this.store.appendAudit({
+          auditId: await stableId(
+            'audit',
+            `${wake.traceId}:wake-retry:${delivery?.attemptCount ?? claim.attemptCount}:${this.store.auditLength()}`,
+          ),
+          traceId: wake.traceId,
+          timestamp: nowIso(this.now),
+          kind: 'wake.retry_scheduled',
+          entityType: 'wake',
+          entityId: wake.wakeId,
+          details: {
+            attemptCount: delivery?.attemptCount ?? claim.attemptCount,
+            nextAttemptAt: delivery?.nextAttemptAt ?? null,
+            message:
+              error instanceof Error
+                ? error.message
+                : 'wake delivery failed',
+          },
+        });
+        return {
+          status: 'retry_scheduled',
+          wake: this.store.latestWake(wake.wakeId) ?? wake,
+        };
+      }
+
       const deadLetter = WakeRecordSchema.parse({
-        ...queuedWake,
+        ...wake,
         createdAt: nowIso(this.now),
         status: 'dead_letter',
       });
       await this.store.appendWake(deadLetter);
       await this.store.appendAudit({
-        auditId: await stableId('audit', `${traceId}:wake-dead-letter`),
-        traceId,
+        auditId: await stableId(
+          'audit',
+          `${wake.traceId}:wake-dead-letter:${this.store.auditLength()}`,
+        ),
+        traceId: wake.traceId,
         timestamp: nowIso(this.now),
         kind: 'wake.dead_letter',
         entityType: 'wake',
-        entityId: wakeId,
+        entityId: wake.wakeId,
         fromState: 'wake_queued',
         toState: 'dead_letter',
         details: {
-          message: error instanceof Error ? error.message : 'wake delivery failed',
+          attemptCount: delivery?.attemptCount ?? claim.attemptCount,
+          message:
+            error instanceof Error
+              ? error.message
+              : 'wake delivery failed',
         },
       });
-
       return {
-        traceId,
-        sourceEventId: request.event.eventId,
         status: 'dead_letter',
-        decision,
         wake: deadLetter,
       };
     }
   }
 
+  async retryWake(wakeId) {
+    const delivery = this.store.getWakeDelivery(wakeId);
+    if (!delivery || delivery.sourceType !== 'event') {
+      return { status: 'not_event_wake', wake: null };
+    }
+
+    const latest = this.store.latestWake(wakeId);
+    if (!latest) {
+      throw new Error(`Unknown wake: ${wakeId}`);
+    }
+    if (latest.status === 'handled') {
+      return { status: 'handled', wake: latest };
+    }
+    if (latest.status === 'dead_letter') {
+      return { status: 'dead_letter', wake: latest };
+    }
+
+    const trace = this.store.trace(delivery.sourceId);
+    const eventRecord = trace.events[0];
+    if (!eventRecord?.lineage || !eventRecord?.event) {
+      const error = new Error(
+        `Cannot reconstruct event wake ${wakeId}; trace ${delivery.sourceId} is missing`,
+      );
+      error.code = 'WAKE_RETRY_SOURCE_MISSING';
+      throw error;
+    }
+
+    const decision = latest.decisionId
+      ? trace.decisions.find(
+          (candidate) => candidate.decisionId === latest.decisionId,
+        ) ?? null
+      : null;
+
+    const queued = WakeRecordSchema.parse({
+      ...latest,
+      status: 'queued',
+      runtimeReceiptId: undefined,
+    });
+    return this.deliverQueuedWake({
+      wake: queued,
+      lineage: eventRecord.lineage,
+      event: eventRecord.event,
+      decision,
+    });
+  }
+
   async acknowledgeWake(wakeId, runtimeReceiptId) {
     const latest = this.store.latestWake(wakeId);
     if (!latest) throw new Error('Unknown wake');