@contember/engine-actions 2.2.0-alpha.2 → 2.2.0-alpha.4

package/dist/development/src/ActionsMetrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ActionsMetrics.js","sources":["../../../../packages/engine-actions/src/ActionsMetrics.ts"],"sourcesContent":["import prom, { Registry } from 'prom-client'\n\nconst LABEL_PROJECT = 'contember_project'\ntype Label = typeof LABEL_PROJECT\n\n/**\n * Prometheus metrics for the Actions dispatch subsystem.\n *\n * Two audiences, deliberately kept as cheap in-memory counters/gauges (no table polling):\n *  - system / operator — is the dispatch loop alive and flowing, or stuck?\n *      `workerHeartbeat` (liveness; works even when the queue is empty) + `workerCrashed`.\n *  - userland — are webhook deliveries failing?\n *      `eventsSucceeded` / `deliveryAttemptsFailed` / `eventsFailed`.\n *\n * Counting units matter: a *delivery attempt* and an *event* are different. A success is always\n * terminal (one event = one successful attempt), but a failure splits — a failed attempt may be\n * retried many times, while a terminally failed event gives up once. Hence the asymmetric set:\n * one success counter, but both a per-attempt failure counter and a terminal-event failure counter.\n *\n * Queue backlog is *estimated* as a trend from the counters:\n *   enqueued − succeeded − failed   (compare rates; the absolute value drifts across restarts and\n *   ignores manual retry/stop, so for an exact depth query the `actions_event` table instead).\n *\n * Labelled only by `contember_project` to keep cardinality bounded; per-target / per-trigger\n * detail belongs in the table-backed userland view, not in Prometheus.\n */\nexport class ActionsMetrics {\n\t/** Events written to the queue (inflow). Unit: event. */\n\tpublic readonly eventsEnqueued: prom.Counter<Label>\n\t/** Events delivered successfully (terminal). Unit: event (= successful attempt). */\n\tpublic readonly eventsSucceeded: prom.Counter<Label>\n\t/** Failed webhook delivery attempts, including ones that will be retried. Unit: attempt. */\n\tpublic readonly deliveryAttemptsFailed: prom.Counter<Label>\n\t/** Events that reached a terminal failed state (retries exhausted or unknown target). Unit: event. */\n\tpublic readonly eventsFailed: prom.Counter<Label>\n\t/** Unix timestamp (seconds) of the last dispatch-loop iteration; staleness ⇒ stuck/dead worker. */\n\tpublic readonly workerHeartbeat: prom.Gauge<Label>\n\t/** Dispatch-loop crashes (each is auto-restarted by the supervisor). */\n\tpublic readonly workerCrashed: prom.Counter<Label>\n\n\tconstructor(registry: Registry) {\n\t\tconst labelNames: Label[] = [LABEL_PROJECT]\n\t\tthis.eventsEnqueued = new prom.Counter({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_events_enqueued_total',\n\t\t\thelp: 'Total number of action events written to the queue (inflow), by contember_project.',\n\t\t\tlabelNames,\n\t\t})\n\t\tthis.eventsSucceeded = new prom.Counter({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_events_succeeded_total',\n\t\t\thelp: 'Total number of action events delivered successfully, by contember_project.',\n\t\t\tlabelNames,\n\t\t})\n\t\tthis.deliveryAttemptsFailed = new prom.Counter({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_delivery_attempts_failed_total',\n\t\t\thelp: 'Total number of failed webhook delivery attempts (including attempts that will be retried), by contember_project.',\n\t\t\tlabelNames,\n\t\t})\n\t\tthis.eventsFailed = new prom.Counter({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_events_failed_total',\n\t\t\thelp: 'Total number of action events that reached a terminal failed state (retries exhausted or unknown target), by contember_project.',\n\t\t\tlabelNames,\n\t\t})\n\t\tthis.workerHeartbeat = new prom.Gauge({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_worker_heartbeat_timestamp_seconds',\n\t\t\thelp: 'Unix timestamp of the last dispatch-loop iteration per contember_project; staleness indicates a stuck or dead worker.',\n\t\t\tlabelNames,\n\t\t})\n\t\tthis.workerCrashed = new prom.Counter({\n\t\t\tregisters: [registry],\n\t\t\tname: 'contember_actions_worker_crashed_total',\n\t\t\thelp: 'Total number of dispatch-loop crashes per contember_project (each is auto-restarted by the supervisor).',\n\t\t\tlabelNames,\n\t\t})\n\t}\n\n\tpublic forProject(projectSlug: string): ProjectActionsMetrics {\n\t\treturn new ProjectActionsMetrics(this, projectSlug)\n\t}\n}\n\n/**\n * A view of {@link ActionsMetrics} bound to a single project, so call sites don't repeat the label.\n */\nexport class ProjectActionsMetrics {\n\tprivate readonly labels: { [LABEL_PROJECT]: string }\n\n\tconstructor(\n\t\tprivate readonly metrics: ActionsMetrics,\n\t\tprivate readonly projectSlug: string,\n\t) {\n\t\tthis.labels = { [LABEL_PROJECT]: projectSlug }\n\t}\n\n\tpublic enqueued(count: number): void {\n\t\tif (count > 0) {\n\t\t\tthis.metrics.eventsEnqueued.inc(this.labels, count)\n\t\t}\n\t}\n\n\tpublic succeeded(count: number): void {\n\t\tif (count > 0) {\n\t\t\tthis.metrics.eventsSucceeded.inc(this.labels, count)\n\t\t}\n\t}\n\n\tpublic deliveryAttemptFailed(count: number): void {\n\t\tif (count > 0) {\n\t\t\tthis.metrics.deliveryAttemptsFailed.inc(this.labels, count)\n\t\t}\n\t}\n\n\tpublic failed(count: number): void {\n\t\tif (count > 0) {\n\t\t\tthis.metrics.eventsFailed.inc(this.labels, count)\n\t\t}\n\t}\n\n\tpublic heartbeat(): void {\n\t\tthis.metrics.workerHeartbeat.set(this.labels, Date.now() / 1000)\n\t}\n\n\tpublic crashed(): void {\n\t\tthis.metrics.workerCrashed.inc(this.labels, 1)\n\t}\n\n\t/** Drop the per-project heartbeat series when the worker stops, so a removed project doesn't look stuck. */\n\tpublic dispose(): void {\n\t\tthis.metrics.workerHeartbeat.remove(this.projectSlug)\n\t}\n}\n"],"names":[],"mappings":";;;;AAEA,MAAM,gBAAgB;AAwBf,MAAM,eAAe;AAAA,EAc3B,YAAY,UAAoB;AAZhC,kBAAA,MAAgB,gBAAA;AAEhB,kBAAA,MAAgB,iBAAA;AAEhB,kBAAA,MAAgB,wBAAA;AAEhB,kBAAA,MAAgB,cAAA;AAEhB,kBAAA,MAAgB,iBAAA;AAEhB,kBAAA,MAAgB,eAAA;AAGf,UAAM,aAAsB,CAAC,aAAa;AAC1C,SAAK,iBAAiB,IAAI,KAAK,QAAQ;AAAA,MACtC,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AACD,SAAK,kBAAkB,IAAI,KAAK,QAAQ;AAAA,MACvC,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AACD,SAAK,yBAAyB,IAAI,KAAK,QAAQ;AAAA,MAC9C,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AACD,SAAK,eAAe,IAAI,KAAK,QAAQ;AAAA,MACpC,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AACD,SAAK,kBAAkB,IAAI,KAAK,MAAM;AAAA,MACrC,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AACD,SAAK,gBAAgB,IAAI,KAAK,QAAQ;AAAA,MACrC,WAAW,CAAC,QAAQ;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IAAA,CACA;AAAA,EACF;AAAA,EAEO,WAAW,aAA4C;AAC7D,WAAO,IAAI,sBAAsB,MAAM,WAAW;AAAA,EACnD;AACD;AAKO,MAAM,sBAAsB;AAAA,EAGlC,YACkB,SACA,aAChB;AAFgB,SAAA,UAAA;AACA,SAAA,cAAA;AAJlB,kBAAA,MAAiB,QAAA;AAMhB,SAAK,SAAS,EAAE,CAAC,aAAa,GAAG,YAAA;AAAA,EAClC;AAAA,EAEO,SAAS,OAAqB;AACpC,QAAI,QAAQ,GAAG;AACd,WAAK,QAAQ,eAAe,IAAI,KAAK,QAAQ,KAAK;AAAA,IACnD;AAAA,EACD;AAAA,EAEO,UAAU,OAAqB;AACrC,QAAI,QAAQ,GAAG;AACd,WAAK,QAAQ,gBAAgB,IAAI,KAAK,QAAQ,KAAK;AAAA,IACpD;AAAA,EACD;AAAA,EAEO,sBAAsB,OAAqB;AACjD,QAAI,QAAQ,GAAG;AACd,WAAK,QAAQ,uBAAuB,IAAI,KAAK,QAAQ,KAAK;AAAA,IAC3D;AAAA,EACD;AAAA,EAEO,OAAO,OAAqB;AAClC,QAAI,QAAQ,GAAG;AACd,WAAK,QAAQ,aAAa,IAAI,KAAK,QAAQ,KAAK;AAAA,IACjD;AAAA,EACD;AAAA,EAEO,YAAkB;AACxB,SAAK,QAAQ,gBAAgB,IAAI,KAAK,QAAQ,KAAK,IAAA,IAAQ,GAAI;AAAA,EAChE;AAAA,EAEO,UAAgB;AACtB,SAAK,QAAQ,cAAc,IAAI,KAAK,QAAQ,CAAC;AAAA,EAC9C;AAAA;AAAA,EAGO,UAAgB;AACtB,SAAK,QAAQ,gBAAgB,OAAO,KAAK,WAAW;AAAA,EACrD;AACD;"}

package/dist/development/src/dispatch/EventDispatcher.cjs

@@ -6,16 +6,22 @@ class EventDispatcher {
     this.variablesManager = variablesManager;
     this.invokeResolver = invokeResolver;
   }
-  async processBatch({ db, contentSchemaResolver, logger }) {
+  async processBatch({ db, contentSchemaResolver, logger, project }) {
     const schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema;
-    const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client);
     const batchId = Math.random().toString().substring(2);
     const batchLogger = logger.child({ loc: "Actions.EventDispatcher", batchId });
+    const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client, batchLogger);
     if (!batch.ok) {
       batchLogger.debug("Nothing to process", {
         backoffMs: batch.backoffMs ?? "undefined"
       });
-      return { failed: 0, succeed: 0, backoffMs: batch.backoffMs };
+      return {
+        succeeded: 0,
+        retried: 0,
+        failedAfterAttempt: 0,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: batch.backoffMs
+      };
     }
     const { target, events } = batch;
     try {
@@ -23,19 +29,31 @@ class EventDispatcher {
       batchLogger.debug("Processing started", {
         events: batch.events.map((it) => it.id)
       });
-      const variables = await this.variablesManager.fetchVariables(db);
+      const variables = await this.variablesManager.fetchVariables(db, project.slug);
       const handledEvents = await handler.handle({ target, events, logger: batchLogger, variables });
-      const [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents);
-      batchLogger.debug("Processing done", { succeed, failed });
-      return { succeed, failed, backoffMs: 0 };
+      const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, handledEvents, batchLogger);
+      batchLogger.debug("Processing done", { succeed: succeeded, failed });
+      return {
+        succeeded,
+        retried,
+        failedAfterAttempt: failed,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: 0
+      };
     } catch (e) {
       logger.error(e, { loc: "EventDispatcher", batchId });
       const failedEvents = events.map((it) => ({
         row: it,
         result: { ok: false, errorMessage: `Internal error` }
       }));
-      await this.eventsRepository.persistProcessed(db.client, failedEvents);
-      return { succeed: 0, failed: batch.events.length, backoffMs: 0 };
+      const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, failedEvents, batchLogger);
+      return {
+        succeeded,
+        retried,
+        failedAfterAttempt: failed,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: 0
+      };
     }
   }
 }

package/dist/development/src/dispatch/EventDispatcher.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"EventDispatcher.cjs","sources":["../../../../../packages/engine-actions/src/dispatch/EventDispatcher.ts"],"sourcesContent":["import { EventsRepository } from './EventsRepository.js'\nimport { TargetHandlerResolver } from './TargetHandlerResolver.js'\nimport { HandledEvent } from './types.js'\nimport { Logger } from '@contember/logger'\nimport { ContentSchemaResolver } from '@contember/engine-http'\nimport { DatabaseContext } from '@contember/engine-system-api'\nimport { VariablesManager } from '../model/VariablesManager.js'\n\ntype ProcessBatchArgs = {\n\tdb: DatabaseContext\n\tcontentSchemaResolver: ContentSchemaResolver\n\tlogger: Logger\n}\n\ntype ProcessBatchResult = {\n\tsucceed: number\n\tfailed: number\n\tbackoffMs: number | undefined\n}\n\nexport class EventDispatcher {\n\tconstructor(\n\t\tprivate readonly eventsRepository: EventsRepository,\n\t\tprivate readonly variablesManager: VariablesManager,\n\t\tprivate readonly invokeResolver: TargetHandlerResolver,\n\t) {\n\t}\n\n\tasync processBatch({ db, contentSchemaResolver, logger }: ProcessBatchArgs): Promise<ProcessBatchResult> {\n\t\tconst schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema\n\t\tconst batch = await this.eventsRepository.fetchBatch(schema.actions, db.client)\n\t\tconst batchId = Math.random().toString().substring(2)\n\t\tconst batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })\n\t\tif (!batch.ok) {\n\t\t\tbatchLogger.debug('Nothing to process', {\n\t\t\t\tbackoffMs: batch.backoffMs ?? 'undefined',\n\t\t\t})\n\t\t\treturn { failed: 0, succeed: 0, backoffMs: batch.backoffMs }\n\t\t}\n\t\tconst { target, events } = batch\n\t\ttry {\n\t\t\tconst handler = this.invokeResolver.resolveHandler(target)\n\t\t\tbatchLogger.debug('Processing started', {\n\t\t\t\tevents: batch.events.map(it => it.id),\n\t\t\t})\n\t\t\tconst variables = await this.variablesManager.fetchVariables(db)\n\t\t\tconst handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })\n\t\t\tconst [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents)\n\t\t\tbatchLogger.debug('Processing done', { succeed, failed })\n\t\t\treturn { succeed, failed, backoffMs: 0 }\n\t\t} catch (e) {\n\t\t\tlogger.error(e, { loc: 'EventDispatcher', batchId })\n\t\t\tconst failedEvents = events.map((it): HandledEvent => ({\n\t\t\t\trow: it,\n\t\t\t\tresult: { ok: false, errorMessage: `Internal error` },\n\t\t\t}))\n\t\t\tawait this.eventsRepository.persistProcessed(db.client, failedEvents)\n\n\t\t\treturn { succeed: 0, failed: batch.events.length, backoffMs: 0 }\n\t\t}\n\t}\n}\n"],"names":[],"mappings":";;AAoBO,MAAM,gBAAgB;AAAA,EAC5B,YACkB,kBACA,kBACA,gBAChB;AAHgB,SAAA,mBAAA;AACA,SAAA,mBAAA;AACA,SAAA,iBAAA;AAAA,EAElB;AAAA,EAEA,MAAM,aAAa,EAAE,IAAI,uBAAuB,UAAyD;AACxG,UAAM,UAAU,MAAM,sBAAsB,UAAU,EAAE,IAAI,WAAW,KAAA,CAAM,GAAG;AAChF,UAAM,QAAQ,MAAM,KAAK,iBAAiB,WAAW,OAAO,SAAS,GAAG,MAAM;AAC9E,UAAM,UAAU,KAAK,OAAA,EAAS,SAAA,EAAW,UAAU,CAAC;AACpD,UAAM,cAAc,OAAO,MAAM,EAAE,KAAK,2BAA2B,SAAS;AAC5E,QAAI,CAAC,MAAM,IAAI;AACd,kBAAY,MAAM,sBAAsB;AAAA,QACvC,WAAW,MAAM,aAAa;AAAA,MAAA,CAC9B;AACD,aAAO,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,MAAM,UAAA;AAAA,IAClD;AACA,UAAM,EAAE,QAAQ,OAAA,IAAW;AAC3B,QAAI;AACH,YAAM,UAAU,KAAK,eAAe,eAAe,MAAM;AACzD,kBAAY,MAAM,sBAAsB;AAAA,QACvC,QAAQ,MAAM,OAAO,IAAI,CAAA,OAAM,GAAG,EAAE;AAAA,MAAA,CACpC;AACD,YAAM,YAAY,MAAM,KAAK,iBAAiB,eAAe,EAAE;AAC/D,YAAM,gBAAgB,MAAM,QAAQ,OAAO,EAAE,QAAQ,QAAQ,QAAQ,aAAa,WAAW;AAC7F,YAAM,CAAC,SAAS,MAAM,IAAI,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,aAAa;AAC/F,kBAAY,MAAM,mBAAmB,EAAE,SAAS,QAAQ;AACxD,aAAO,EAAE,SAAS,QAAQ,WAAW,EAAA;AAAA,IACtC,SAAS,GAAG;AACX,aAAO,MAAM,GAAG,EAAE,KAAK,mBAAmB,SAAS;AACnD,YAAM,eAAe,OAAO,IAAI,CAAC,QAAsB;AAAA,QACtD,KAAK;AAAA,QACL,QAAQ,EAAE,IAAI,OAAO,cAAc,iBAAA;AAAA,MAAiB,EACnD;AACF,YAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,YAAY;AAEpE,aAAO,EAAE,SAAS,GAAG,QAAQ,MAAM,OAAO,QAAQ,WAAW,EAAA;AAAA,IAC9D;AAAA,EACD;AACD;;"}
+{"version":3,"file":"EventDispatcher.cjs","sources":["../../../../../packages/engine-actions/src/dispatch/EventDispatcher.ts"],"sourcesContent":["import { EventsRepository } from './EventsRepository.js'\nimport { TargetHandlerResolver } from './TargetHandlerResolver.js'\nimport { HandledEvent } from './types.js'\nimport { Logger } from '@contember/logger'\nimport { ContentSchemaResolver } from '@contember/engine-http'\nimport { DatabaseContext } from '@contember/engine-system-api'\nimport { VariablesManager } from '../model/VariablesManager.js'\n\ntype ProcessBatchArgs = {\n\tdb: DatabaseContext\n\tcontentSchemaResolver: ContentSchemaResolver\n\tlogger: Logger\n\tproject: { slug: string }\n}\n\ntype ProcessBatchResult = {\n\t/** Events delivered successfully (terminal). */\n\tsucceeded: number\n\t/** Failed delivery attempts that will be retried. */\n\tretried: number\n\t/** Events terminally failed after a delivery attempt (retries exhausted). */\n\tfailedAfterAttempt: number\n\t/** Events terminally failed without a delivery attempt (target missing from schema). */\n\tfailedUnknownTarget: number\n\tbackoffMs: number | undefined\n}\n\nexport class EventDispatcher {\n\tconstructor(\n\t\tprivate readonly eventsRepository: EventsRepository,\n\t\tprivate readonly variablesManager: VariablesManager,\n\t\tprivate readonly invokeResolver: TargetHandlerResolver,\n\t) {\n\t}\n\n\tasync processBatch({ db, contentSchemaResolver, logger, project }: ProcessBatchArgs): Promise<ProcessBatchResult> {\n\t\tconst schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema\n\t\tconst batchId = Math.random().toString().substring(2)\n\t\tconst batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })\n\t\tconst batch = await this.eventsRepository.fetchBatch(schema.actions, db.client, batchLogger)\n\t\tif (!batch.ok) {\n\t\t\tbatchLogger.debug('Nothing to process', {\n\t\t\t\tbackoffMs: batch.backoffMs ?? 'undefined',\n\t\t\t})\n\t\t\treturn {\n\t\t\t\tsucceeded: 0,\n\t\t\t\tretried: 0,\n\t\t\t\tfailedAfterAttempt: 0,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: batch.backoffMs,\n\t\t\t}\n\t\t}\n\t\tconst { target, events } = batch\n\t\ttry {\n\t\t\tconst handler = this.invokeResolver.resolveHandler(target)\n\t\t\tbatchLogger.debug('Processing started', {\n\t\t\t\tevents: batch.events.map(it => it.id),\n\t\t\t})\n\t\t\tconst variables = await this.variablesManager.fetchVariables(db, project.slug)\n\t\t\tconst handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })\n\t\t\tconst { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, handledEvents, batchLogger)\n\t\t\tbatchLogger.debug('Processing done', { succeed: succeeded, failed })\n\t\t\treturn {\n\t\t\t\tsucceeded,\n\t\t\t\tretried,\n\t\t\t\tfailedAfterAttempt: failed,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: 0,\n\t\t\t}\n\t\t} catch (e) {\n\t\t\tlogger.error(e, { loc: 'EventDispatcher', batchId })\n\t\t\tconst failedEvents = events.map((it): HandledEvent => ({\n\t\t\t\trow: it,\n\t\t\t\tresult: { ok: false, errorMessage: `Internal error` },\n\t\t\t}))\n\t\t\tconst { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, failedEvents, batchLogger)\n\n\t\t\treturn {\n\t\t\t\tsucceeded,\n\t\t\t\tretried,\n\t\t\t\tfailedAfterAttempt: failed,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: 0,\n\t\t\t}\n\t\t}\n\t}\n}\n"],"names":[],"mappings":";;AA2BO,MAAM,gBAAgB;AAAA,EAC5B,YACkB,kBACA,kBACA,gBAChB;AAHgB,SAAA,mBAAA;AACA,SAAA,mBAAA;AACA,SAAA,iBAAA;AAAA,EAElB;AAAA,EAEA,MAAM,aAAa,EAAE,IAAI,uBAAuB,QAAQ,WAA0D;AACjH,UAAM,UAAU,MAAM,sBAAsB,UAAU,EAAE,IAAI,WAAW,KAAA,CAAM,GAAG;AAChF,UAAM,UAAU,KAAK,OAAA,EAAS,SAAA,EAAW,UAAU,CAAC;AACpD,UAAM,cAAc,OAAO,MAAM,EAAE,KAAK,2BAA2B,SAAS;AAC5E,UAAM,QAAQ,MAAM,KAAK,iBAAiB,WAAW,OAAO,SAAS,GAAG,QAAQ,WAAW;AAC3F,QAAI,CAAC,MAAM,IAAI;AACd,kBAAY,MAAM,sBAAsB;AAAA,QACvC,WAAW,MAAM,aAAa;AAAA,MAAA,CAC9B;AACD,aAAO;AAAA,QACN,WAAW;AAAA,QACX,SAAS;AAAA,QACT,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW,MAAM;AAAA,MAAA;AAAA,IAEnB;AACA,UAAM,EAAE,QAAQ,OAAA,IAAW;AAC3B,QAAI;AACH,YAAM,UAAU,KAAK,eAAe,eAAe,MAAM;AACzD,kBAAY,MAAM,sBAAsB;AAAA,QACvC,QAAQ,MAAM,OAAO,IAAI,CAAA,OAAM,GAAG,EAAE;AAAA,MAAA,CACpC;AACD,YAAM,YAAY,MAAM,KAAK,iBAAiB,eAAe,IAAI,QAAQ,IAAI;AAC7E,YAAM,gBAAgB,MAAM,QAAQ,OAAO,EAAE,QAAQ,QAAQ,QAAQ,aAAa,WAAW;AAC7F,YAAM,EAAE,WAAW,SAAS,OAAA,IAAW,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,eAAe,WAAW;AACzH,kBAAY,MAAM,mBAAmB,EAAE,SAAS,WAAW,QAAQ;AACnE,aAAO;AAAA,QACN;AAAA,QACA;AAAA,QACA,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW;AAAA,MAAA;AAAA,IAEb,SAAS,GAAG;AACX,aAAO,MAAM,GAAG,EAAE,KAAK,mBAAmB,SAAS;AACnD,YAAM,eAAe,OAAO,IAAI,CAAC,QAAsB;AAAA,QACtD,KAAK;AAAA,QACL,QAAQ,EAAE,IAAI,OAAO,cAAc,iBAAA;AAAA,MAAiB,EACnD;AACF,YAAM,EAAE,WAAW,SAAS,OAAA,IAAW,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,cAAc,WAAW;AAExH,aAAO;AAAA,QACN;AAAA,QACA;AAAA,QACA,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW;AAAA,MAAA;AAAA,IAEb;AAAA,EACD;AACD;;"}

package/dist/development/src/dispatch/EventDispatcher.js

@@ -4,16 +4,22 @@ class EventDispatcher {
     this.variablesManager = variablesManager;
     this.invokeResolver = invokeResolver;
   }
-  async processBatch({ db, contentSchemaResolver, logger }) {
+  async processBatch({ db, contentSchemaResolver, logger, project }) {
     const schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema;
-    const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client);
     const batchId = Math.random().toString().substring(2);
     const batchLogger = logger.child({ loc: "Actions.EventDispatcher", batchId });
+    const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client, batchLogger);
     if (!batch.ok) {
       batchLogger.debug("Nothing to process", {
         backoffMs: batch.backoffMs ?? "undefined"
       });
-      return { failed: 0, succeed: 0, backoffMs: batch.backoffMs };
+      return {
+        succeeded: 0,
+        retried: 0,
+        failedAfterAttempt: 0,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: batch.backoffMs
+      };
     }
     const { target, events } = batch;
     try {
@@ -21,19 +27,31 @@ class EventDispatcher {
       batchLogger.debug("Processing started", {
         events: batch.events.map((it) => it.id)
       });
-      const variables = await this.variablesManager.fetchVariables(db);
+      const variables = await this.variablesManager.fetchVariables(db, project.slug);
       const handledEvents = await handler.handle({ target, events, logger: batchLogger, variables });
-      const [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents);
-      batchLogger.debug("Processing done", { succeed, failed });
-      return { succeed, failed, backoffMs: 0 };
+      const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, handledEvents, batchLogger);
+      batchLogger.debug("Processing done", { succeed: succeeded, failed });
+      return {
+        succeeded,
+        retried,
+        failedAfterAttempt: failed,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: 0
+      };
     } catch (e) {
       logger.error(e, { loc: "EventDispatcher", batchId });
       const failedEvents = events.map((it) => ({
         row: it,
         result: { ok: false, errorMessage: `Internal error` }
       }));
-      await this.eventsRepository.persistProcessed(db.client, failedEvents);
-      return { succeed: 0, failed: batch.events.length, backoffMs: 0 };
+      const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, failedEvents, batchLogger);
+      return {
+        succeeded,
+        retried,
+        failedAfterAttempt: failed,
+        failedUnknownTarget: batch.unknownTargetFailed,
+        backoffMs: 0
+      };
     }
   }
 }

package/dist/development/src/dispatch/EventDispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"EventDispatcher.js","sources":["../../../../../packages/engine-actions/src/dispatch/EventDispatcher.ts"],"sourcesContent":["import { EventsRepository } from './EventsRepository.js'\nimport { TargetHandlerResolver } from './TargetHandlerResolver.js'\nimport { HandledEvent } from './types.js'\nimport { Logger } from '@contember/logger'\nimport { ContentSchemaResolver } from '@contember/engine-http'\nimport { DatabaseContext } from '@contember/engine-system-api'\nimport { VariablesManager } from '../model/VariablesManager.js'\n\ntype ProcessBatchArgs = {\n\tdb: DatabaseContext\n\tcontentSchemaResolver: ContentSchemaResolver\n\tlogger: Logger\n}\n\ntype ProcessBatchResult = {\n\tsucceed: number\n\tfailed: number\n\tbackoffMs: number | undefined\n}\n\nexport class EventDispatcher {\n\tconstructor(\n\t\tprivate readonly eventsRepository: EventsRepository,\n\t\tprivate readonly variablesManager: VariablesManager,\n\t\tprivate readonly invokeResolver: TargetHandlerResolver,\n\t) {\n\t}\n\n\tasync processBatch({ db, contentSchemaResolver, logger }: ProcessBatchArgs): Promise<ProcessBatchResult> {\n\t\tconst schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema\n\t\tconst batch = await this.eventsRepository.fetchBatch(schema.actions, db.client)\n\t\tconst batchId = Math.random().toString().substring(2)\n\t\tconst batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })\n\t\tif (!batch.ok) {\n\t\t\tbatchLogger.debug('Nothing to process', {\n\t\t\t\tbackoffMs: batch.backoffMs ?? 'undefined',\n\t\t\t})\n\t\t\treturn { failed: 0, succeed: 0, backoffMs: batch.backoffMs }\n\t\t}\n\t\tconst { target, events } = batch\n\t\ttry {\n\t\t\tconst handler = this.invokeResolver.resolveHandler(target)\n\t\t\tbatchLogger.debug('Processing started', {\n\t\t\t\tevents: batch.events.map(it => it.id),\n\t\t\t})\n\t\t\tconst variables = await this.variablesManager.fetchVariables(db)\n\t\t\tconst handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })\n\t\t\tconst [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents)\n\t\t\tbatchLogger.debug('Processing done', { succeed, failed })\n\t\t\treturn { succeed, failed, backoffMs: 0 }\n\t\t} catch (e) {\n\t\t\tlogger.error(e, { loc: 'EventDispatcher', batchId })\n\t\t\tconst failedEvents = events.map((it): HandledEvent => ({\n\t\t\t\trow: it,\n\t\t\t\tresult: { ok: false, errorMessage: `Internal error` },\n\t\t\t}))\n\t\t\tawait this.eventsRepository.persistProcessed(db.client, failedEvents)\n\n\t\t\treturn { succeed: 0, failed: batch.events.length, backoffMs: 0 }\n\t\t}\n\t}\n}\n"],"names":[],"mappings":"AAoBO,MAAM,gBAAgB;AAAA,EAC5B,YACkB,kBACA,kBACA,gBAChB;AAHgB,SAAA,mBAAA;AACA,SAAA,mBAAA;AACA,SAAA,iBAAA;AAAA,EAElB;AAAA,EAEA,MAAM,aAAa,EAAE,IAAI,uBAAuB,UAAyD;AACxG,UAAM,UAAU,MAAM,sBAAsB,UAAU,EAAE,IAAI,WAAW,KAAA,CAAM,GAAG;AAChF,UAAM,QAAQ,MAAM,KAAK,iBAAiB,WAAW,OAAO,SAAS,GAAG,MAAM;AAC9E,UAAM,UAAU,KAAK,OAAA,EAAS,SAAA,EAAW,UAAU,CAAC;AACpD,UAAM,cAAc,OAAO,MAAM,EAAE,KAAK,2BAA2B,SAAS;AAC5E,QAAI,CAAC,MAAM,IAAI;AACd,kBAAY,MAAM,sBAAsB;AAAA,QACvC,WAAW,MAAM,aAAa;AAAA,MAAA,CAC9B;AACD,aAAO,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,MAAM,UAAA;AAAA,IAClD;AACA,UAAM,EAAE,QAAQ,OAAA,IAAW;AAC3B,QAAI;AACH,YAAM,UAAU,KAAK,eAAe,eAAe,MAAM;AACzD,kBAAY,MAAM,sBAAsB;AAAA,QACvC,QAAQ,MAAM,OAAO,IAAI,CAAA,OAAM,GAAG,EAAE;AAAA,MAAA,CACpC;AACD,YAAM,YAAY,MAAM,KAAK,iBAAiB,eAAe,EAAE;AAC/D,YAAM,gBAAgB,MAAM,QAAQ,OAAO,EAAE,QAAQ,QAAQ,QAAQ,aAAa,WAAW;AAC7F,YAAM,CAAC,SAAS,MAAM,IAAI,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,aAAa;AAC/F,kBAAY,MAAM,mBAAmB,EAAE,SAAS,QAAQ;AACxD,aAAO,EAAE,SAAS,QAAQ,WAAW,EAAA;AAAA,IACtC,SAAS,GAAG;AACX,aAAO,MAAM,GAAG,EAAE,KAAK,mBAAmB,SAAS;AACnD,YAAM,eAAe,OAAO,IAAI,CAAC,QAAsB;AAAA,QACtD,KAAK;AAAA,QACL,QAAQ,EAAE,IAAI,OAAO,cAAc,iBAAA;AAAA,MAAiB,EACnD;AACF,YAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,YAAY;AAEpE,aAAO,EAAE,SAAS,GAAG,QAAQ,MAAM,OAAO,QAAQ,WAAW,EAAA;AAAA,IAC9D;AAAA,EACD;AACD;"}
+{"version":3,"file":"EventDispatcher.js","sources":["../../../../../packages/engine-actions/src/dispatch/EventDispatcher.ts"],"sourcesContent":["import { EventsRepository } from './EventsRepository.js'\nimport { TargetHandlerResolver } from './TargetHandlerResolver.js'\nimport { HandledEvent } from './types.js'\nimport { Logger } from '@contember/logger'\nimport { ContentSchemaResolver } from '@contember/engine-http'\nimport { DatabaseContext } from '@contember/engine-system-api'\nimport { VariablesManager } from '../model/VariablesManager.js'\n\ntype ProcessBatchArgs = {\n\tdb: DatabaseContext\n\tcontentSchemaResolver: ContentSchemaResolver\n\tlogger: Logger\n\tproject: { slug: string }\n}\n\ntype ProcessBatchResult = {\n\t/** Events delivered successfully (terminal). */\n\tsucceeded: number\n\t/** Failed delivery attempts that will be retried. */\n\tretried: number\n\t/** Events terminally failed after a delivery attempt (retries exhausted). */\n\tfailedAfterAttempt: number\n\t/** Events terminally failed without a delivery attempt (target missing from schema). */\n\tfailedUnknownTarget: number\n\tbackoffMs: number | undefined\n}\n\nexport class EventDispatcher {\n\tconstructor(\n\t\tprivate readonly eventsRepository: EventsRepository,\n\t\tprivate readonly variablesManager: VariablesManager,\n\t\tprivate readonly invokeResolver: TargetHandlerResolver,\n\t) {\n\t}\n\n\tasync processBatch({ db, contentSchemaResolver, logger, project }: ProcessBatchArgs): Promise<ProcessBatchResult> {\n\t\tconst schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema\n\t\tconst batchId = Math.random().toString().substring(2)\n\t\tconst batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })\n\t\tconst batch = await this.eventsRepository.fetchBatch(schema.actions, db.client, batchLogger)\n\t\tif (!batch.ok) {\n\t\t\tbatchLogger.debug('Nothing to process', {\n\t\t\t\tbackoffMs: batch.backoffMs ?? 'undefined',\n\t\t\t})\n\t\t\treturn {\n\t\t\t\tsucceeded: 0,\n\t\t\t\tretried: 0,\n\t\t\t\tfailedAfterAttempt: 0,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: batch.backoffMs,\n\t\t\t}\n\t\t}\n\t\tconst { target, events } = batch\n\t\ttry {\n\t\t\tconst handler = this.invokeResolver.resolveHandler(target)\n\t\t\tbatchLogger.debug('Processing started', {\n\t\t\t\tevents: batch.events.map(it => it.id),\n\t\t\t})\n\t\t\tconst variables = await this.variablesManager.fetchVariables(db, project.slug)\n\t\t\tconst handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })\n\t\t\tconst { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, handledEvents, batchLogger)\n\t\t\tbatchLogger.debug('Processing done', { succeed: succeeded, failed })\n\t\t\treturn {\n\t\t\t\tsucceeded,\n\t\t\t\tretried,\n\t\t\t\tfailedAfterAttempt: failed,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: 0,\n\t\t\t}\n\t\t} catch (e) {\n\t\t\tlogger.error(e, { loc: 'EventDispatcher', batchId })\n\t\t\tconst failedEvents = events.map((it): HandledEvent => ({\n\t\t\t\trow: it,\n\t\t\t\tresult: { ok: false, errorMessage: `Internal error` },\n\t\t\t}))\n\t\t\tconst { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, failedEvents, batchLogger)\n\n\t\t\treturn {\n\t\t\t\tsucceeded,\n\t\t\t\tretried,\n\t\t\t\tfailedAfterAttempt: failed,\n\t\t\t\tfailedUnknownTarget: batch.unknownTargetFailed,\n\t\t\t\tbackoffMs: 0,\n\t\t\t}\n\t\t}\n\t}\n}\n"],"names":[],"mappings":"AA2BO,MAAM,gBAAgB;AAAA,EAC5B,YACkB,kBACA,kBACA,gBAChB;AAHgB,SAAA,mBAAA;AACA,SAAA,mBAAA;AACA,SAAA,iBAAA;AAAA,EAElB;AAAA,EAEA,MAAM,aAAa,EAAE,IAAI,uBAAuB,QAAQ,WAA0D;AACjH,UAAM,UAAU,MAAM,sBAAsB,UAAU,EAAE,IAAI,WAAW,KAAA,CAAM,GAAG;AAChF,UAAM,UAAU,KAAK,OAAA,EAAS,SAAA,EAAW,UAAU,CAAC;AACpD,UAAM,cAAc,OAAO,MAAM,EAAE,KAAK,2BAA2B,SAAS;AAC5E,UAAM,QAAQ,MAAM,KAAK,iBAAiB,WAAW,OAAO,SAAS,GAAG,QAAQ,WAAW;AAC3F,QAAI,CAAC,MAAM,IAAI;AACd,kBAAY,MAAM,sBAAsB;AAAA,QACvC,WAAW,MAAM,aAAa;AAAA,MAAA,CAC9B;AACD,aAAO;AAAA,QACN,WAAW;AAAA,QACX,SAAS;AAAA,QACT,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW,MAAM;AAAA,MAAA;AAAA,IAEnB;AACA,UAAM,EAAE,QAAQ,OAAA,IAAW;AAC3B,QAAI;AACH,YAAM,UAAU,KAAK,eAAe,eAAe,MAAM;AACzD,kBAAY,MAAM,sBAAsB;AAAA,QACvC,QAAQ,MAAM,OAAO,IAAI,CAAA,OAAM,GAAG,EAAE;AAAA,MAAA,CACpC;AACD,YAAM,YAAY,MAAM,KAAK,iBAAiB,eAAe,IAAI,QAAQ,IAAI;AAC7E,YAAM,gBAAgB,MAAM,QAAQ,OAAO,EAAE,QAAQ,QAAQ,QAAQ,aAAa,WAAW;AAC7F,YAAM,EAAE,WAAW,SAAS,OAAA,IAAW,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,eAAe,WAAW;AACzH,kBAAY,MAAM,mBAAmB,EAAE,SAAS,WAAW,QAAQ;AACnE,aAAO;AAAA,QACN;AAAA,QACA;AAAA,QACA,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW;AAAA,MAAA;AAAA,IAEb,SAAS,GAAG;AACX,aAAO,MAAM,GAAG,EAAE,KAAK,mBAAmB,SAAS;AACnD,YAAM,eAAe,OAAO,IAAI,CAAC,QAAsB;AAAA,QACtD,KAAK;AAAA,QACL,QAAQ,EAAE,IAAI,OAAO,cAAc,iBAAA;AAAA,MAAiB,EACnD;AACF,YAAM,EAAE,WAAW,SAAS,OAAA,IAAW,MAAM,KAAK,iBAAiB,iBAAiB,GAAG,QAAQ,cAAc,WAAW;AAExH,aAAO;AAAA,QACN;AAAA,QACA;AAAA,QACA,oBAAoB;AAAA,QACpB,qBAAqB,MAAM;AAAA,QAC3B,WAAW;AAAA,MAAA;AAAA,IAEb;AAAA,EACD;AACD;"}

package/dist/development/src/dispatch/EventsRepository.cjs

@@ -7,19 +7,29 @@ const DEFAULT_REPEAT_INTERVAL_MS = 5e3;
 const DEFAULT_MAX_ATTEMPTS = 10;
 const DEFAULT_BATCH_SIZE = 1;
 class EventsRepository {
-  async fetchBatch(actions, db) {
-    const primaryEvent = (await this.fetchInternal(db, 1))[0];
-    if (!primaryEvent) {
-      return { ok: false, backoffMs: await this.fetchBackOff(db) };
-    }
-    const target = actions.targets[primaryEvent.target];
-    if (!target) {
-      await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id);
-      return await this.fetchBatch(actions, db);
+  async fetchBatch(actions, db, logger) {
+    let unknownTargetFailed = 0;
+    while (true) {
+      const primaryEvent = (await this.fetchInternal(db, 1))[0];
+      if (!primaryEvent) {
+        return { ok: false, backoffMs: await this.fetchBackOff(db), unknownTargetFailed };
+      }
+      const target = actions.targets[primaryEvent.target];
+      if (!target) {
+        if (await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)) {
+          unknownTargetFailed++;
+          logger.error("Action event failed permanently, its target is not in the schema", {
+            eventId: primaryEvent.id,
+            target: primaryEvent.target,
+            trigger: primaryEvent.trigger
+          });
+        }
+        continue;
+      }
+      const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1;
+      const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : [];
+      return { ok: true, events: [primaryEvent, ...batch], target, unknownTargetFailed };
     }
-    const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1;
-    const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : [];
-    return { ok: true, events: [primaryEvent, ...batch], target };
   }
   async fetchBackOff(db) {
     const result = await Database.SelectBuilder.create().from("actions_event").match(EventsToProcessSpecification.eventsToProcessStateSpecification).select((it) => it.raw("ceil(extract(epoch from visible_at - now()) * 1000)"), "ms").orderBy("visible_at", "asc").limit(1).getResult(db);
@@ -34,17 +44,32 @@ class EventsRepository {
       visible_at: visibleAt
     }).from((it) => it.from("events").where((it2) => it2.columnsEq(["events", "id"], ["actions_event", "id"]))).returning(new Database.Literal("*")).execute(db);
   }
-  async persistProcessed(db, events) {
+  async persistProcessed(db, events, logger) {
     return await db.transaction(async (trx) => {
       await trx.query(Database.Connection.REPEATABLE_READ);
       const succeed = events.filter((it) => it.result.ok);
       await this.markSucceed(trx, succeed.map((it) => it.row));
+      let retried = 0;
+      let failed = 0;
       for (const event of events) {
         if (!event.result.ok) {
-          await this.markFailed(trx, event);
+          const terminal = await this.markFailed(trx, event);
+          if (terminal) {
+            failed++;
+            logger.error("Action event failed permanently, no attempts left", {
+              eventId: event.row.id,
+              target: event.row.target,
+              trigger: event.row.trigger,
+              attempts: event.row.num_retries + 1,
+              code: event.result.code,
+              errorMessage: event.result.errorMessage
+            });
+          } else {
+            retried++;
+          }
         }
       }
-      return [succeed.length, events.length - succeed.length];
+      return { succeeded: succeed.length, retried, failed };
     });
   }
   async requeue(db, events) {
@@ -80,6 +105,7 @@ class EventsRepository {
       last_state_change: now
     }).where((it) => it.in("id", rows.map((it2) => it2.id))).execute(db);
   }
+  /** Returns `true` when the event reached a terminal `failed` state, `false` when it will be retried. */
   async markFailed(db, event) {
     const numRetries = event.row.num_retries + 1;
     const now = /* @__PURE__ */ new Date();
@@ -87,23 +113,25 @@ class EventsRepository {
     nextAttempt.setTime(
       nextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries)
     );
+    const terminal = numRetries >= (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
     await Database.UpdateBuilder.create().table("actions_event").values({
       last_state_change: now,
       num_retries: numRetries,
       log: JSON.stringify([...event.row.log, event.result]),
-      ...numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS) ? {
+      ...terminal ? {
+        state: "failed"
+      } : {
         state: "retrying",
         visible_at: nextAttempt
-      } : {
-        state: "failed"
       }
     }).where({ id: event.row.id }).execute(db);
+    return terminal;
   }
   async markFailedOnUnknownTarget(db, targetName, primaryEventId) {
-    await Database.UpdateBuilder.create().table("actions_event").where(
+    const affectedRows = await Database.UpdateBuilder.create().table("actions_event").where(
       (it) => it.or(
         (it2) => it2.and(
-          (it3) => it3.in("state", ["retrying", "created", "processing"]).compare("visible_at", Database.Operator.lte, "now").compare("target", Database.Operator.eq, targetName)
+          (it3) => it3.in("state", ["processing"]).compare("target", Database.Operator.eq, targetName)
         ).and(
           (it3) => it3.compare("id", Database.Operator.eq, primaryEventId)
         )
@@ -116,6 +144,7 @@ class EventsRepository {
       state: "failed",
       resolved_at: /* @__PURE__ */ new Date()
     }).execute(db);
+    return affectedRows > 0;
   }
 }
 exports.EventsRepository = EventsRepository;

package/dist/development/src/dispatch/EventsRepository.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"EventsRepository.cjs","sources":["../../../../../packages/engine-actions/src/dispatch/EventsRepository.ts"],"sourcesContent":["import { Client, Connection, Literal, LockModifier, LockType, Operator, SelectBuilder, UpdateBuilder } from '@contember/database'\nimport { EventRow, HandledEvent } from './types.js'\nimport { Actions } from '@contember/schema'\nimport { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification.js'\nimport { notify } from '../utils/notifyChannel.js'\n\nconst ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes\nconst DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds\nconst DEFAULT_MAX_ATTEMPTS = 10\nconst DEFAULT_BATCH_SIZE = 1\n\nexport class EventsRepository {\n\tpublic async fetchBatch(actions: Actions.Schema, db: Client): Promise<FetchBatchResult> {\n\t\tconst primaryEvent = (await this.fetchInternal(db, 1))[0]\n\t\tif (!primaryEvent) {\n\t\t\treturn { ok: false, backoffMs: await this.fetchBackOff(db) }\n\t\t}\n\t\tconst target = actions.targets[primaryEvent.target]\n\t\tif (!target) {\n\t\t\tawait this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)\n\t\t\treturn await this.fetchBatch(actions, db)\n\t\t}\n\t\tconst batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1\n\t\tconst batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []\n\t\treturn { ok: true, events: [primaryEvent, ...batch], target }\n\t}\n\n\tprivate async fetchBackOff(db: Client): Promise<number | undefined> {\n\t\tconst result = await SelectBuilder.create<{ ms: number }>()\n\t\t\t.from('actions_event')\n\t\t\t.match(eventsToProcessStateSpecification)\n\t\t\t.select(it => it.raw('ceil(extract(epoch from visible_at - now()) * 1000)'), 'ms')\n\t\t\t.orderBy('visible_at', 'asc')\n\t\t\t.limit(1)\n\t\t\t.getResult(db)\n\n\t\treturn result.length ? Math.max(0, result[0].ms) : undefined\n\t}\n\n\tprivate async fetchInternal(db: Client, limit: number, targetName?: string): Promise<EventRow[]> {\n\t\tconst visibleAt = new Date()\n\t\tvisibleAt.setTime(visibleAt.getTime() + ACK_TIMEOUT_MS)\n\t\treturn await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.with('events', qb =>\n\t\t\t\tqb\n\t\t\t\t\t.select(it => it.raw('*'))\n\t\t\t\t\t.from('actions_event')\n\t\t\t\t\t.match(eventsToProcessSpecification)\n\t\t\t\t\t.where(targetName ? { target: targetName } : {})\n\t\t\t\t\t.limit(limit)\n\t\t\t\t\t.lock(LockType.forNoKeyUpdate, LockModifier.skipLocked))\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'processing',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: visibleAt,\n\t\t\t})\n\t\t\t.from(it => it.from('events').where(it => it.columnsEq(['events', 'id'], ['actions_event', 'id'])))\n\t\t\t.returning<EventRow>(new Literal('*'))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async persistProcessed(db: Client, events: HandledEvent[]): Promise<[number, number]> {\n\t\treturn await db.transaction(async trx => {\n\t\t\tawait trx.query(Connection.REPEATABLE_READ)\n\t\t\tconst succeed = events.filter(it => it.result.ok)\n\t\t\tawait this.markSucceed(trx, succeed.map(it => it.row))\n\n\t\t\tfor (const event of events) {\n\t\t\t\tif (!event.result.ok) {\n\t\t\t\t\tawait this.markFailed(trx, event)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn [succeed.length, events.length - succeed.length]\n\t\t})\n\t}\n\n\tpublic async requeue(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: it =>\n\t\t\t\t\tit.raw(\n\t\t\t\t\t\t`\n\t\t\t\t\tCASE \n\t\t\t\t\tWHEN num_retries = 0 THEN \n\t\t\t\t\t\t?::__SCHEMA__.actions_trigger_state \n\t\t\t\t\tELSE \n\t\t\t\t\t?::__SCHEMA__.actions_trigger_state END\n\t\t\t\t`,\n\t\t\t\t\t\t'created',\n\t\t\t\t\t\t'retrying',\n\t\t\t\t\t),\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async markStopped(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'stopped',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markSucceed(db: Client, rows: EventRow[]): Promise<void> {\n\t\tconst now = new Date()\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'succeed',\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tresolved_at: now,\n\t\t\t\tlast_state_change: now,\n\t\t\t})\n\t\t\t.where(it => it.in('id', rows.map(it => it.id)))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markFailed(db: Client, event: HandledEvent): Promise<void> {\n\t\tconst numRetries = event.row.num_retries + 1\n\t\tconst now = new Date()\n\t\tconst nextAttempt = new Date()\n\t\tnextAttempt.setTime(\n\t\t\tnextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries),\n\t\t)\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tlast_state_change: now,\n\t\t\t\tnum_retries: numRetries,\n\t\t\t\tlog: JSON.stringify([...event.row.log, event.result]) as any,\n\t\t\t\t...(numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)\n\t\t\t\t\t? {\n\t\t\t\t\t\tstate: 'retrying',\n\t\t\t\t\t\tvisible_at: nextAttempt,\n\t\t\t\t\t}\n\t\t\t\t\t: {\n\t\t\t\t\t\tstate: 'failed',\n\t\t\t\t\t}),\n\t\t\t})\n\t\t\t.where({ id: event.row.id })\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string) {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.where(it =>\n\t\t\t\tit.or(it =>\n\t\t\t\t\tit\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.in('state', ['retrying', 'created', 'processing'])\n\t\t\t\t\t\t\t\t.compare('visible_at', Operator.lte, 'now')\n\t\t\t\t\t\t\t\t.compare('target', Operator.eq, targetName)\n\t\t\t\t\t\t)\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.compare('id', Operator.eq, primaryEventId)\n\t\t\t\t\t\t)\n\t\t\t\t)\n\t\t\t)\n\t\t\t.values<EventRow>({\n\t\t\t\t// intentionally not retrying\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tlog: it => it.raw('log || ?::jsonb', { ok: false, errorMessage: `Event target ${targetName} not found, stopping.` }),\n\t\t\t\tstate: 'failed',\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.execute(db)\n\t}\n}\n\nexport type FetchBatchResult =\n\t| { ok: true; events: EventRow[]; target: Actions.AnyTarget }\n\t| { ok: false; backoffMs?: number }\n"],"names":["SelectBuilder","eventsToProcessStateSpecification","UpdateBuilder","eventsToProcessSpecification","LockType","LockModifier","it","Literal","Connection","Operator"],"mappings":";;;;AAMA,MAAM,iBAAiB,MAAQ,KAAK;AACpC,MAAM,6BAA6B;AACnC,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAEpB,MAAM,iBAAiB;AAAA,EAC7B,MAAa,WAAW,SAAyB,IAAuC;AACvF,UAAM,gBAAgB,MAAM,KAAK,cAAc,IAAI,CAAC,GAAG,CAAC;AACxD,QAAI,CAAC,cAAc;AAClB,aAAO,EAAE,IAAI,OAAO,WAAW,MAAM,KAAK,aAAa,EAAE,EAAA;AAAA,IAC1D;AACA,UAAM,SAAS,QAAQ,QAAQ,aAAa,MAAM;AAClD,QAAI,CAAC,QAAQ;AACZ,YAAM,KAAK,0BAA0B,IAAI,aAAa,QAAQ,aAAa,EAAE;AAC7E,aAAO,MAAM,KAAK,WAAW,SAAS,EAAE;AAAA,IACzC;AACA,UAAM,aAAa,OAAO,aAAa,sBAAsB;AAC7D,UAAM,QAAQ,YAAY,IAAI,MAAM,KAAK,cAAc,IAAI,SAAS,IAAI,CAAA;AACxE,WAAO,EAAE,IAAI,MAAM,QAAQ,CAAC,cAAc,GAAG,KAAK,GAAG,OAAA;AAAA,EACtD;AAAA,EAEA,MAAc,aAAa,IAAyC;AACnE,UAAM,SAAS,MAAMA,uBAAc,OAAA,EACjC,KAAK,eAAe,EACpB,MAAMC,6BAAAA,iCAAiC,EACvC,OAAO,CAAA,OAAM,GAAG,IAAI,qDAAqD,GAAG,IAAI,EAChF,QAAQ,cAAc,KAAK,EAC3B,MAAM,CAAC,EACP,UAAU,EAAE;AAEd,WAAO,OAAO,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI;AAAA,EACpD;AAAA,EAEA,MAAc,cAAc,IAAY,OAAe,YAA0C;AAChG,UAAM,gCAAgB,KAAA;AACtB,cAAU,QAAQ,UAAU,QAAA,IAAY,cAAc;AACtD,WAAO,MAAMC,SAAAA,cAAc,OAAA,EACzB,MAAM,eAAe,EACrB,KAAK,UAAU,QACf,GACE,OAAO,QAAM,GAAG,IAAI,GAAG,CAAC,EACxB,KAAK,eAAe,EACpB,MAAMC,6BAAAA,4BAA4B,EAClC,MAAM,aAAa,EAAE,QAAQ,WAAA,IAAe,EAAE,EAC9C,MAAM,KAAK,EACX,KAAKC,kBAAS,gBAAgBC,SAAAA,aAAa,UAAU,CAAC,EACxD,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,YAAY;AAAA,IAAA,CACZ,EACA,KAAK,CAAA,OAAM,GAAG,KAAK,QAAQ,EAAE,MAAM,CAAAC,QAAMA,IAAG,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,EACjG,UAAoB,IAAIC,SAAAA,QAAQ,GAAG,CAAC,EACpC,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,iBAAiB,IAAY,QAAmD;AAC5F,WAAO,MAAM,GAAG,YAAY,OAAM,QAAO;AACxC,YAAM,IAAI,MAAMC,SAAAA,WAAW,eAAe;AAC1C,YAAM,UAAU,OAAO,OAAO,CAAA,OAAM,GAAG,OAAO,EAAE;AAChD,YAAM,KAAK,YAAY,KAAK,QAAQ,IAAI,CAAA,OAAM,GAAG,GAAG,CAAC;AAErD,iBAAW,SAAS,QAAQ;AAC3B,YAAI,CAAC,MAAM,OAAO,IAAI;AACrB,gBAAM,KAAK,WAAW,KAAK,KAAK;AAAA,QACjC;AAAA,MACD;AACA,aAAO,CAAC,QAAQ,QAAQ,OAAO,SAAS,QAAQ,MAAM;AAAA,IACvD,CAAC;AAAA,EACF;AAAA,EAEA,MAAa,QAAQ,IAAY,QAAiC;AACjE,UAAMN,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO,QACN,GAAG;AAAA,QACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,QACA;AAAA,MAAA;AAAA,MAEF,uCAAuB,KAAA;AAAA,MACvB,gCAAgB,KAAA;AAAA,IAAK,CACrB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,YAAY,IAAY,QAAiC;AACrE,UAAMA,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,YAAY,IAAY,MAAiC;AACtE,UAAM,0BAAU,KAAA;AAChB,UAAMA,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,aAAa;AAAA,MACb,mBAAmB;AAAA,IAAA,CACnB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,KAAK,IAAI,CAAAI,QAAMA,IAAG,EAAE,CAAC,CAAC,EAC9C,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,WAAW,IAAY,OAAoC;AACxE,UAAM,aAAa,MAAM,IAAI,cAAc;AAC3C,UAAM,0BAAU,KAAA;AAChB,UAAM,kCAAkB,KAAA;AACxB,gBAAY;AAAA,MACX,YAAY,QAAA,KAAa,MAAM,QAAQ,2BAA2B,8BAA8B,KAAK,IAAI,GAAG,MAAM,IAAI,WAAW;AAAA,IAAA;AAElI,UAAMJ,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,mBAAmB;AAAA,MACnB,aAAa;AAAA,MACb,KAAK,KAAK,UAAU,CAAC,GAAG,MAAM,IAAI,KAAK,MAAM,MAAM,CAAC;AAAA,MACpD,GAAI,cAAc,MAAM,QAAQ,eAAe,wBAC5C;AAAA,QACD,OAAO;AAAA,QACP,YAAY;AAAA,MAAA,IAEX;AAAA,QACD,OAAO;AAAA,MAAA;AAAA,IACR,CACD,EACA,MAAM,EAAE,IAAI,MAAM,IAAI,GAAA,CAAI,EAC1B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,0BAA0B,IAAY,YAAoB,gBAAwB;AAC/F,UAAMA,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB;AAAA,MAAM,QACN,GAAG;AAAA,QAAG,CAAAI,QACLA,IACE;AAAA,UAAI,CAAAA,QACJA,IACE,GAAG,SAAS,CAAC,YAAY,WAAW,YAAY,CAAC,EACjD,QAAQ,cAAcG,SAAAA,SAAS,KAAK,KAAK,EACzC,QAAQ,UAAUA,kBAAS,IAAI,UAAU;AAAA,QAAA,EAE3C;AAAA,UAAI,CAAAH,QACJA,IACE,QAAQ,MAAMG,SAAAA,SAAS,IAAI,cAAc;AAAA,QAAA;AAAA,MAC5C;AAAA,IACF,EAEA,OAAiB;AAAA;AAAA,MAEjB,uCAAuB,KAAA;AAAA,MACvB,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,KAAK,CAAA,OAAM,GAAG,IAAI,mBAAmB,EAAE,IAAI,OAAO,cAAc,gBAAgB,UAAU,wBAAA,CAAyB;AAAA,MACnH,OAAO;AAAA,MACP,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,QAAQ,EAAE;AAAA,EACb;AACD;;"}
+{"version":3,"file":"EventsRepository.cjs","sources":["../../../../../packages/engine-actions/src/dispatch/EventsRepository.ts"],"sourcesContent":["import { Client, Connection, Literal, LockModifier, LockType, Operator, SelectBuilder, UpdateBuilder } from '@contember/database'\nimport { EventRow, HandledEvent } from './types.js'\nimport { Actions } from '@contember/schema'\nimport { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification.js'\nimport { notify } from '../utils/notifyChannel.js'\nimport { Logger } from '@contember/logger'\n\nconst ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes\nconst DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds\nconst DEFAULT_MAX_ATTEMPTS = 10\nconst DEFAULT_BATCH_SIZE = 1\n\nexport class EventsRepository {\n\tpublic async fetchBatch(actions: Actions.Schema, db: Client, logger: Logger): Promise<FetchBatchResult> {\n\t\t// Events referencing a target that no longer exists in the schema are terminally failed and\n\t\t// skipped over; count them so the caller can report them as terminal failures.\n\t\tlet unknownTargetFailed = 0\n\t\twhile (true) {\n\t\t\tconst primaryEvent = (await this.fetchInternal(db, 1))[0]\n\t\t\tif (!primaryEvent) {\n\t\t\t\treturn { ok: false, backoffMs: await this.fetchBackOff(db), unknownTargetFailed }\n\t\t\t}\n\t\t\tconst target = actions.targets[primaryEvent.target]\n\t\t\tif (!target) {\n\t\t\t\tif (await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)) {\n\t\t\t\t\tunknownTargetFailed++\n\t\t\t\t\t// Never retried: the schema has to name the target again before such an event can be delivered.\n\t\t\t\t\tlogger.error('Action event failed permanently, its target is not in the schema', {\n\t\t\t\t\t\teventId: primaryEvent.id,\n\t\t\t\t\t\ttarget: primaryEvent.target,\n\t\t\t\t\t\ttrigger: primaryEvent.trigger,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tconst batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1\n\t\t\tconst batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []\n\t\t\treturn { ok: true, events: [primaryEvent, ...batch], target, unknownTargetFailed }\n\t\t}\n\t}\n\n\tprivate async fetchBackOff(db: Client): Promise<number | undefined> {\n\t\tconst result = await SelectBuilder.create<{ ms: number }>()\n\t\t\t.from('actions_event')\n\t\t\t.match(eventsToProcessStateSpecification)\n\t\t\t.select(it => it.raw('ceil(extract(epoch from visible_at - now()) * 1000)'), 'ms')\n\t\t\t.orderBy('visible_at', 'asc')\n\t\t\t.limit(1)\n\t\t\t.getResult(db)\n\n\t\treturn result.length ? Math.max(0, result[0].ms) : undefined\n\t}\n\n\tprivate async fetchInternal(db: Client, limit: number, targetName?: string): Promise<EventRow[]> {\n\t\tconst visibleAt = new Date()\n\t\tvisibleAt.setTime(visibleAt.getTime() + ACK_TIMEOUT_MS)\n\t\treturn await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.with('events', qb =>\n\t\t\t\tqb\n\t\t\t\t\t.select(it => it.raw('*'))\n\t\t\t\t\t.from('actions_event')\n\t\t\t\t\t.match(eventsToProcessSpecification)\n\t\t\t\t\t.where(targetName ? { target: targetName } : {})\n\t\t\t\t\t.limit(limit)\n\t\t\t\t\t.lock(LockType.forNoKeyUpdate, LockModifier.skipLocked))\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'processing',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: visibleAt,\n\t\t\t})\n\t\t\t.from(it => it.from('events').where(it => it.columnsEq(['events', 'id'], ['actions_event', 'id'])))\n\t\t\t.returning<EventRow>(new Literal('*'))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async persistProcessed(db: Client, events: HandledEvent[], logger: Logger): Promise<PersistProcessedResult> {\n\t\treturn await db.transaction(async trx => {\n\t\t\tawait trx.query(Connection.REPEATABLE_READ)\n\t\t\tconst succeed = events.filter(it => it.result.ok)\n\t\t\tawait this.markSucceed(trx, succeed.map(it => it.row))\n\n\t\t\tlet retried = 0\n\t\t\tlet failed = 0\n\t\t\tfor (const event of events) {\n\t\t\t\tif (!event.result.ok) {\n\t\t\t\t\tconst terminal = await this.markFailed(trx, event)\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\tfailed++\n\t\t\t\t\t\t// Nothing else reports a given-up event; the target is named, never described by URL or headers.\n\t\t\t\t\t\tlogger.error('Action event failed permanently, no attempts left', {\n\t\t\t\t\t\t\teventId: event.row.id,\n\t\t\t\t\t\t\ttarget: event.row.target,\n\t\t\t\t\t\t\ttrigger: event.row.trigger,\n\t\t\t\t\t\t\tattempts: event.row.num_retries + 1,\n\t\t\t\t\t\t\tcode: event.result.code,\n\t\t\t\t\t\t\terrorMessage: event.result.errorMessage,\n\t\t\t\t\t\t})\n\t\t\t\t\t} else {\n\t\t\t\t\t\tretried++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn { succeeded: succeed.length, retried, failed }\n\t\t})\n\t}\n\n\tpublic async requeue(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: it =>\n\t\t\t\t\tit.raw(\n\t\t\t\t\t\t`\n\t\t\t\t\tCASE \n\t\t\t\t\tWHEN num_retries = 0 THEN \n\t\t\t\t\t\t?::__SCHEMA__.actions_trigger_state \n\t\t\t\t\tELSE \n\t\t\t\t\t?::__SCHEMA__.actions_trigger_state END\n\t\t\t\t`,\n\t\t\t\t\t\t'created',\n\t\t\t\t\t\t'retrying',\n\t\t\t\t\t),\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async markStopped(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'stopped',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markSucceed(db: Client, rows: EventRow[]): Promise<void> {\n\t\tconst now = new Date()\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'succeed',\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tresolved_at: now,\n\t\t\t\tlast_state_change: now,\n\t\t\t})\n\t\t\t.where(it => it.in('id', rows.map(it => it.id)))\n\t\t\t.execute(db)\n\t}\n\n\t/** Returns `true` when the event reached a terminal `failed` state, `false` when it will be retried. */\n\tprivate async markFailed(db: Client, event: HandledEvent): Promise<boolean> {\n\t\tconst numRetries = event.row.num_retries + 1\n\t\tconst now = new Date()\n\t\tconst nextAttempt = new Date()\n\t\tnextAttempt.setTime(\n\t\t\tnextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries),\n\t\t)\n\t\tconst terminal = numRetries >= (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tlast_state_change: now,\n\t\t\t\tnum_retries: numRetries,\n\t\t\t\tlog: JSON.stringify([...event.row.log, event.result]) as any,\n\t\t\t\t...(terminal\n\t\t\t\t\t? {\n\t\t\t\t\t\tstate: 'failed',\n\t\t\t\t\t}\n\t\t\t\t\t: {\n\t\t\t\t\t\tstate: 'retrying',\n\t\t\t\t\t\tvisible_at: nextAttempt,\n\t\t\t\t\t}),\n\t\t\t})\n\t\t\t.where({ id: event.row.id })\n\t\t\t.execute(db)\n\t\treturn terminal\n\t}\n\n\tprivate async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string): Promise<boolean> {\n\t\tconst affectedRows = await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.where(it =>\n\t\t\t\tit.or(it =>\n\t\t\t\t\tit\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.in('state', ['processing'])\n\t\t\t\t\t\t\t\t.compare('target', Operator.eq, targetName)\n\t\t\t\t\t\t)\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.compare('id', Operator.eq, primaryEventId)\n\t\t\t\t\t\t)\n\t\t\t\t)\n\t\t\t)\n\t\t\t.values<EventRow>({\n\t\t\t\t// intentionally not retrying\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tlog: it => it.raw('log || ?::jsonb', { ok: false, errorMessage: `Event target ${targetName} not found, stopping.` }),\n\t\t\t\tstate: 'failed',\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.execute(db)\n\t\treturn affectedRows > 0\n\t}\n}\n\nexport type FetchBatchResult =\n\t| { ok: true; events: EventRow[]; target: Actions.AnyTarget; unknownTargetFailed: number }\n\t| { ok: false; backoffMs?: number; unknownTargetFailed: number }\n\nexport type PersistProcessedResult = {\n\t/** Events delivered successfully (terminal). */\n\tsucceeded: number\n\t/** Events whose delivery attempt failed and that will be retried later. */\n\tretried: number\n\t/** Events whose delivery attempt failed and that reached a terminal `failed` state. */\n\tfailed: number\n}\n"],"names":["SelectBuilder","eventsToProcessStateSpecification","UpdateBuilder","eventsToProcessSpecification","LockType","LockModifier","it","Literal","Connection","Operator"],"mappings":";;;;AAOA,MAAM,iBAAiB,MAAQ,KAAK;AACpC,MAAM,6BAA6B;AACnC,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAEpB,MAAM,iBAAiB;AAAA,EAC7B,MAAa,WAAW,SAAyB,IAAY,QAA2C;AAGvG,QAAI,sBAAsB;AAC1B,WAAO,MAAM;AACZ,YAAM,gBAAgB,MAAM,KAAK,cAAc,IAAI,CAAC,GAAG,CAAC;AACxD,UAAI,CAAC,cAAc;AAClB,eAAO,EAAE,IAAI,OAAO,WAAW,MAAM,KAAK,aAAa,EAAE,GAAG,oBAAA;AAAA,MAC7D;AACA,YAAM,SAAS,QAAQ,QAAQ,aAAa,MAAM;AAClD,UAAI,CAAC,QAAQ;AACZ,YAAI,MAAM,KAAK,0BAA0B,IAAI,aAAa,QAAQ,aAAa,EAAE,GAAG;AACnF;AAEA,iBAAO,MAAM,oEAAoE;AAAA,YAChF,SAAS,aAAa;AAAA,YACtB,QAAQ,aAAa;AAAA,YACrB,SAAS,aAAa;AAAA,UAAA,CACtB;AAAA,QACF;AACA;AAAA,MACD;AACA,YAAM,aAAa,OAAO,aAAa,sBAAsB;AAC7D,YAAM,QAAQ,YAAY,IAAI,MAAM,KAAK,cAAc,IAAI,SAAS,IAAI,CAAA;AACxE,aAAO,EAAE,IAAI,MAAM,QAAQ,CAAC,cAAc,GAAG,KAAK,GAAG,QAAQ,oBAAA;AAAA,IAC9D;AAAA,EACD;AAAA,EAEA,MAAc,aAAa,IAAyC;AACnE,UAAM,SAAS,MAAMA,uBAAc,OAAA,EACjC,KAAK,eAAe,EACpB,MAAMC,6BAAAA,iCAAiC,EACvC,OAAO,CAAA,OAAM,GAAG,IAAI,qDAAqD,GAAG,IAAI,EAChF,QAAQ,cAAc,KAAK,EAC3B,MAAM,CAAC,EACP,UAAU,EAAE;AAEd,WAAO,OAAO,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI;AAAA,EACpD;AAAA,EAEA,MAAc,cAAc,IAAY,OAAe,YAA0C;AAChG,UAAM,gCAAgB,KAAA;AACtB,cAAU,QAAQ,UAAU,QAAA,IAAY,cAAc;AACtD,WAAO,MAAMC,SAAAA,cAAc,OAAA,EACzB,MAAM,eAAe,EACrB,KAAK,UAAU,QACf,GACE,OAAO,QAAM,GAAG,IAAI,GAAG,CAAC,EACxB,KAAK,eAAe,EACpB,MAAMC,6BAAAA,4BAA4B,EAClC,MAAM,aAAa,EAAE,QAAQ,WAAA,IAAe,EAAE,EAC9C,MAAM,KAAK,EACX,KAAKC,kBAAS,gBAAgBC,SAAAA,aAAa,UAAU,CAAC,EACxD,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,YAAY;AAAA,IAAA,CACZ,EACA,KAAK,CAAA,OAAM,GAAG,KAAK,QAAQ,EAAE,MAAM,CAAAC,QAAMA,IAAG,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,EACjG,UAAoB,IAAIC,SAAAA,QAAQ,GAAG,CAAC,EACpC,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,iBAAiB,IAAY,QAAwB,QAAiD;AAClH,WAAO,MAAM,GAAG,YAAY,OAAM,QAAO;AACxC,YAAM,IAAI,MAAMC,SAAAA,WAAW,eAAe;AAC1C,YAAM,UAAU,OAAO,OAAO,CAAA,OAAM,GAAG,OAAO,EAAE;AAChD,YAAM,KAAK,YAAY,KAAK,QAAQ,IAAI,CAAA,OAAM,GAAG,GAAG,CAAC;AAErD,UAAI,UAAU;AACd,UAAI,SAAS;AACb,iBAAW,SAAS,QAAQ;AAC3B,YAAI,CAAC,MAAM,OAAO,IAAI;AACrB,gBAAM,WAAW,MAAM,KAAK,WAAW,KAAK,KAAK;AACjD,cAAI,UAAU;AACb;AAEA,mBAAO,MAAM,qDAAqD;AAAA,cACjE,SAAS,MAAM,IAAI;AAAA,cACnB,QAAQ,MAAM,IAAI;AAAA,cAClB,SAAS,MAAM,IAAI;AAAA,cACnB,UAAU,MAAM,IAAI,cAAc;AAAA,cAClC,MAAM,MAAM,OAAO;AAAA,cACnB,cAAc,MAAM,OAAO;AAAA,YAAA,CAC3B;AAAA,UACF,OAAO;AACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AACA,aAAO,EAAE,WAAW,QAAQ,QAAQ,SAAS,OAAA;AAAA,IAC9C,CAAC;AAAA,EACF;AAAA,EAEA,MAAa,QAAQ,IAAY,QAAiC;AACjE,UAAMN,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO,QACN,GAAG;AAAA,QACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,QACA;AAAA,MAAA;AAAA,MAEF,uCAAuB,KAAA;AAAA,MACvB,gCAAgB,KAAA;AAAA,IAAK,CACrB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,YAAY,IAAY,QAAiC;AACrE,UAAMA,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,YAAY,IAAY,MAAiC;AACtE,UAAM,0BAAU,KAAA;AAChB,UAAMA,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,aAAa;AAAA,MACb,mBAAmB;AAAA,IAAA,CACnB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,KAAK,IAAI,CAAAI,QAAMA,IAAG,EAAE,CAAC,CAAC,EAC9C,QAAQ,EAAE;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,WAAW,IAAY,OAAuC;AAC3E,UAAM,aAAa,MAAM,IAAI,cAAc;AAC3C,UAAM,0BAAU,KAAA;AAChB,UAAM,kCAAkB,KAAA;AACxB,gBAAY;AAAA,MACX,YAAY,QAAA,KAAa,MAAM,QAAQ,2BAA2B,8BAA8B,KAAK,IAAI,GAAG,MAAM,IAAI,WAAW;AAAA,IAAA;AAElI,UAAM,WAAW,eAAe,MAAM,QAAQ,eAAe;AAC7D,UAAMJ,SAAAA,cAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,mBAAmB;AAAA,MACnB,aAAa;AAAA,MACb,KAAK,KAAK,UAAU,CAAC,GAAG,MAAM,IAAI,KAAK,MAAM,MAAM,CAAC;AAAA,MACpD,GAAI,WACD;AAAA,QACD,OAAO;AAAA,MAAA,IAEN;AAAA,QACD,OAAO;AAAA,QACP,YAAY;AAAA,MAAA;AAAA,IACb,CACD,EACA,MAAM,EAAE,IAAI,MAAM,IAAI,GAAA,CAAI,EAC1B,QAAQ,EAAE;AACZ,WAAO;AAAA,EACR;AAAA,EAEA,MAAc,0BAA0B,IAAY,YAAoB,gBAA0C;AACjH,UAAM,eAAe,MAAMA,uBAAc,SACvC,MAAM,eAAe,EACrB;AAAA,MAAM,QACN,GAAG;AAAA,QAAG,CAAAI,QACLA,IACE;AAAA,UAAI,CAAAA,QACJA,IACE,GAAG,SAAS,CAAC,YAAY,CAAC,EAC1B,QAAQ,UAAUG,SAAAA,SAAS,IAAI,UAAU;AAAA,QAAA,EAE3C;AAAA,UAAI,CAAAH,QACJA,IACE,QAAQ,MAAMG,SAAAA,SAAS,IAAI,cAAc;AAAA,QAAA;AAAA,MAC5C;AAAA,IACF,EAEA,OAAiB;AAAA;AAAA,MAEjB,uCAAuB,KAAA;AAAA,MACvB,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,KAAK,CAAA,OAAM,GAAG,IAAI,mBAAmB,EAAE,IAAI,OAAO,cAAc,gBAAgB,UAAU,wBAAA,CAAyB;AAAA,MACnH,OAAO;AAAA,MACP,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,QAAQ,EAAE;AACZ,WAAO,eAAe;AAAA,EACvB;AACD;;"}

package/dist/development/src/dispatch/EventsRepository.js

@@ -5,19 +5,29 @@ const DEFAULT_REPEAT_INTERVAL_MS = 5e3;
 const DEFAULT_MAX_ATTEMPTS = 10;
 const DEFAULT_BATCH_SIZE = 1;
 class EventsRepository {
-  async fetchBatch(actions, db) {
-    const primaryEvent = (await this.fetchInternal(db, 1))[0];
-    if (!primaryEvent) {
-      return { ok: false, backoffMs: await this.fetchBackOff(db) };
-    }
-    const target = actions.targets[primaryEvent.target];
-    if (!target) {
-      await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id);
-      return await this.fetchBatch(actions, db);
+  async fetchBatch(actions, db, logger) {
+    let unknownTargetFailed = 0;
+    while (true) {
+      const primaryEvent = (await this.fetchInternal(db, 1))[0];
+      if (!primaryEvent) {
+        return { ok: false, backoffMs: await this.fetchBackOff(db), unknownTargetFailed };
+      }
+      const target = actions.targets[primaryEvent.target];
+      if (!target) {
+        if (await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)) {
+          unknownTargetFailed++;
+          logger.error("Action event failed permanently, its target is not in the schema", {
+            eventId: primaryEvent.id,
+            target: primaryEvent.target,
+            trigger: primaryEvent.trigger
+          });
+        }
+        continue;
+      }
+      const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1;
+      const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : [];
+      return { ok: true, events: [primaryEvent, ...batch], target, unknownTargetFailed };
     }
-    const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1;
-    const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : [];
-    return { ok: true, events: [primaryEvent, ...batch], target };
   }
   async fetchBackOff(db) {
     const result = await SelectBuilder.create().from("actions_event").match(eventsToProcessStateSpecification).select((it) => it.raw("ceil(extract(epoch from visible_at - now()) * 1000)"), "ms").orderBy("visible_at", "asc").limit(1).getResult(db);
@@ -32,17 +42,32 @@ class EventsRepository {
       visible_at: visibleAt
     }).from((it) => it.from("events").where((it2) => it2.columnsEq(["events", "id"], ["actions_event", "id"]))).returning(new Literal("*")).execute(db);
   }
-  async persistProcessed(db, events) {
+  async persistProcessed(db, events, logger) {
     return await db.transaction(async (trx) => {
       await trx.query(Connection.REPEATABLE_READ);
       const succeed = events.filter((it) => it.result.ok);
       await this.markSucceed(trx, succeed.map((it) => it.row));
+      let retried = 0;
+      let failed = 0;
       for (const event of events) {
         if (!event.result.ok) {
-          await this.markFailed(trx, event);
+          const terminal = await this.markFailed(trx, event);
+          if (terminal) {
+            failed++;
+            logger.error("Action event failed permanently, no attempts left", {
+              eventId: event.row.id,
+              target: event.row.target,
+              trigger: event.row.trigger,
+              attempts: event.row.num_retries + 1,
+              code: event.result.code,
+              errorMessage: event.result.errorMessage
+            });
+          } else {
+            retried++;
+          }
         }
       }
-      return [succeed.length, events.length - succeed.length];
+      return { succeeded: succeed.length, retried, failed };
     });
   }
   async requeue(db, events) {
@@ -78,6 +103,7 @@ class EventsRepository {
       last_state_change: now
     }).where((it) => it.in("id", rows.map((it2) => it2.id))).execute(db);
   }
+  /** Returns `true` when the event reached a terminal `failed` state, `false` when it will be retried. */
   async markFailed(db, event) {
     const numRetries = event.row.num_retries + 1;
     const now = /* @__PURE__ */ new Date();
@@ -85,23 +111,25 @@ class EventsRepository {
     nextAttempt.setTime(
       nextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries)
     );
+    const terminal = numRetries >= (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
     await UpdateBuilder$1.create().table("actions_event").values({
       last_state_change: now,
       num_retries: numRetries,
       log: JSON.stringify([...event.row.log, event.result]),
-      ...numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS) ? {
+      ...terminal ? {
+        state: "failed"
+      } : {
         state: "retrying",
         visible_at: nextAttempt
-      } : {
-        state: "failed"
       }
     }).where({ id: event.row.id }).execute(db);
+    return terminal;
   }
   async markFailedOnUnknownTarget(db, targetName, primaryEventId) {
-    await UpdateBuilder$1.create().table("actions_event").where(
+    const affectedRows = await UpdateBuilder$1.create().table("actions_event").where(
       (it) => it.or(
         (it2) => it2.and(
-          (it3) => it3.in("state", ["retrying", "created", "processing"]).compare("visible_at", Operator.lte, "now").compare("target", Operator.eq, targetName)
+          (it3) => it3.in("state", ["processing"]).compare("target", Operator.eq, targetName)
         ).and(
           (it3) => it3.compare("id", Operator.eq, primaryEventId)
         )
@@ -114,6 +142,7 @@ class EventsRepository {
       state: "failed",
       resolved_at: /* @__PURE__ */ new Date()
     }).execute(db);
+    return affectedRows > 0;
   }
 }
 export {

package/dist/development/src/dispatch/EventsRepository.js.map

@@ -1 +1 @@
-{"version":3,"file":"EventsRepository.js","sources":["../../../../../packages/engine-actions/src/dispatch/EventsRepository.ts"],"sourcesContent":["import { Client, Connection, Literal, LockModifier, LockType, Operator, SelectBuilder, UpdateBuilder } from '@contember/database'\nimport { EventRow, HandledEvent } from './types.js'\nimport { Actions } from '@contember/schema'\nimport { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification.js'\nimport { notify } from '../utils/notifyChannel.js'\n\nconst ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes\nconst DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds\nconst DEFAULT_MAX_ATTEMPTS = 10\nconst DEFAULT_BATCH_SIZE = 1\n\nexport class EventsRepository {\n\tpublic async fetchBatch(actions: Actions.Schema, db: Client): Promise<FetchBatchResult> {\n\t\tconst primaryEvent = (await this.fetchInternal(db, 1))[0]\n\t\tif (!primaryEvent) {\n\t\t\treturn { ok: false, backoffMs: await this.fetchBackOff(db) }\n\t\t}\n\t\tconst target = actions.targets[primaryEvent.target]\n\t\tif (!target) {\n\t\t\tawait this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)\n\t\t\treturn await this.fetchBatch(actions, db)\n\t\t}\n\t\tconst batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1\n\t\tconst batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []\n\t\treturn { ok: true, events: [primaryEvent, ...batch], target }\n\t}\n\n\tprivate async fetchBackOff(db: Client): Promise<number | undefined> {\n\t\tconst result = await SelectBuilder.create<{ ms: number }>()\n\t\t\t.from('actions_event')\n\t\t\t.match(eventsToProcessStateSpecification)\n\t\t\t.select(it => it.raw('ceil(extract(epoch from visible_at - now()) * 1000)'), 'ms')\n\t\t\t.orderBy('visible_at', 'asc')\n\t\t\t.limit(1)\n\t\t\t.getResult(db)\n\n\t\treturn result.length ? Math.max(0, result[0].ms) : undefined\n\t}\n\n\tprivate async fetchInternal(db: Client, limit: number, targetName?: string): Promise<EventRow[]> {\n\t\tconst visibleAt = new Date()\n\t\tvisibleAt.setTime(visibleAt.getTime() + ACK_TIMEOUT_MS)\n\t\treturn await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.with('events', qb =>\n\t\t\t\tqb\n\t\t\t\t\t.select(it => it.raw('*'))\n\t\t\t\t\t.from('actions_event')\n\t\t\t\t\t.match(eventsToProcessSpecification)\n\t\t\t\t\t.where(targetName ? { target: targetName } : {})\n\t\t\t\t\t.limit(limit)\n\t\t\t\t\t.lock(LockType.forNoKeyUpdate, LockModifier.skipLocked))\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'processing',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: visibleAt,\n\t\t\t})\n\t\t\t.from(it => it.from('events').where(it => it.columnsEq(['events', 'id'], ['actions_event', 'id'])))\n\t\t\t.returning<EventRow>(new Literal('*'))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async persistProcessed(db: Client, events: HandledEvent[]): Promise<[number, number]> {\n\t\treturn await db.transaction(async trx => {\n\t\t\tawait trx.query(Connection.REPEATABLE_READ)\n\t\t\tconst succeed = events.filter(it => it.result.ok)\n\t\t\tawait this.markSucceed(trx, succeed.map(it => it.row))\n\n\t\t\tfor (const event of events) {\n\t\t\t\tif (!event.result.ok) {\n\t\t\t\t\tawait this.markFailed(trx, event)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn [succeed.length, events.length - succeed.length]\n\t\t})\n\t}\n\n\tpublic async requeue(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: it =>\n\t\t\t\t\tit.raw(\n\t\t\t\t\t\t`\n\t\t\t\t\tCASE \n\t\t\t\t\tWHEN num_retries = 0 THEN \n\t\t\t\t\t\t?::__SCHEMA__.actions_trigger_state \n\t\t\t\t\tELSE \n\t\t\t\t\t?::__SCHEMA__.actions_trigger_state END\n\t\t\t\t`,\n\t\t\t\t\t\t'created',\n\t\t\t\t\t\t'retrying',\n\t\t\t\t\t),\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async markStopped(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'stopped',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markSucceed(db: Client, rows: EventRow[]): Promise<void> {\n\t\tconst now = new Date()\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'succeed',\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tresolved_at: now,\n\t\t\t\tlast_state_change: now,\n\t\t\t})\n\t\t\t.where(it => it.in('id', rows.map(it => it.id)))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markFailed(db: Client, event: HandledEvent): Promise<void> {\n\t\tconst numRetries = event.row.num_retries + 1\n\t\tconst now = new Date()\n\t\tconst nextAttempt = new Date()\n\t\tnextAttempt.setTime(\n\t\t\tnextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries),\n\t\t)\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tlast_state_change: now,\n\t\t\t\tnum_retries: numRetries,\n\t\t\t\tlog: JSON.stringify([...event.row.log, event.result]) as any,\n\t\t\t\t...(numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)\n\t\t\t\t\t? {\n\t\t\t\t\t\tstate: 'retrying',\n\t\t\t\t\t\tvisible_at: nextAttempt,\n\t\t\t\t\t}\n\t\t\t\t\t: {\n\t\t\t\t\t\tstate: 'failed',\n\t\t\t\t\t}),\n\t\t\t})\n\t\t\t.where({ id: event.row.id })\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string) {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.where(it =>\n\t\t\t\tit.or(it =>\n\t\t\t\t\tit\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.in('state', ['retrying', 'created', 'processing'])\n\t\t\t\t\t\t\t\t.compare('visible_at', Operator.lte, 'now')\n\t\t\t\t\t\t\t\t.compare('target', Operator.eq, targetName)\n\t\t\t\t\t\t)\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.compare('id', Operator.eq, primaryEventId)\n\t\t\t\t\t\t)\n\t\t\t\t)\n\t\t\t)\n\t\t\t.values<EventRow>({\n\t\t\t\t// intentionally not retrying\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tlog: it => it.raw('log || ?::jsonb', { ok: false, errorMessage: `Event target ${targetName} not found, stopping.` }),\n\t\t\t\tstate: 'failed',\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.execute(db)\n\t}\n}\n\nexport type FetchBatchResult =\n\t| { ok: true; events: EventRow[]; target: Actions.AnyTarget }\n\t| { ok: false; backoffMs?: number }\n"],"names":["UpdateBuilder","it"],"mappings":";;AAMA,MAAM,iBAAiB,MAAQ,KAAK;AACpC,MAAM,6BAA6B;AACnC,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAEpB,MAAM,iBAAiB;AAAA,EAC7B,MAAa,WAAW,SAAyB,IAAuC;AACvF,UAAM,gBAAgB,MAAM,KAAK,cAAc,IAAI,CAAC,GAAG,CAAC;AACxD,QAAI,CAAC,cAAc;AAClB,aAAO,EAAE,IAAI,OAAO,WAAW,MAAM,KAAK,aAAa,EAAE,EAAA;AAAA,IAC1D;AACA,UAAM,SAAS,QAAQ,QAAQ,aAAa,MAAM;AAClD,QAAI,CAAC,QAAQ;AACZ,YAAM,KAAK,0BAA0B,IAAI,aAAa,QAAQ,aAAa,EAAE;AAC7E,aAAO,MAAM,KAAK,WAAW,SAAS,EAAE;AAAA,IACzC;AACA,UAAM,aAAa,OAAO,aAAa,sBAAsB;AAC7D,UAAM,QAAQ,YAAY,IAAI,MAAM,KAAK,cAAc,IAAI,SAAS,IAAI,CAAA;AACxE,WAAO,EAAE,IAAI,MAAM,QAAQ,CAAC,cAAc,GAAG,KAAK,GAAG,OAAA;AAAA,EACtD;AAAA,EAEA,MAAc,aAAa,IAAyC;AACnE,UAAM,SAAS,MAAM,cAAc,OAAA,EACjC,KAAK,eAAe,EACpB,MAAM,iCAAiC,EACvC,OAAO,CAAA,OAAM,GAAG,IAAI,qDAAqD,GAAG,IAAI,EAChF,QAAQ,cAAc,KAAK,EAC3B,MAAM,CAAC,EACP,UAAU,EAAE;AAEd,WAAO,OAAO,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI;AAAA,EACpD;AAAA,EAEA,MAAc,cAAc,IAAY,OAAe,YAA0C;AAChG,UAAM,gCAAgB,KAAA;AACtB,cAAU,QAAQ,UAAU,QAAA,IAAY,cAAc;AACtD,WAAO,MAAMA,gBAAc,OAAA,EACzB,MAAM,eAAe,EACrB,KAAK,UAAU,QACf,GACE,OAAO,QAAM,GAAG,IAAI,GAAG,CAAC,EACxB,KAAK,eAAe,EACpB,MAAM,4BAA4B,EAClC,MAAM,aAAa,EAAE,QAAQ,WAAA,IAAe,EAAE,EAC9C,MAAM,KAAK,EACX,KAAK,SAAS,gBAAgB,aAAa,UAAU,CAAC,EACxD,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,YAAY;AAAA,IAAA,CACZ,EACA,KAAK,CAAA,OAAM,GAAG,KAAK,QAAQ,EAAE,MAAM,CAAAC,QAAMA,IAAG,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,EACjG,UAAoB,IAAI,QAAQ,GAAG,CAAC,EACpC,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,iBAAiB,IAAY,QAAmD;AAC5F,WAAO,MAAM,GAAG,YAAY,OAAM,QAAO;AACxC,YAAM,IAAI,MAAM,WAAW,eAAe;AAC1C,YAAM,UAAU,OAAO,OAAO,CAAA,OAAM,GAAG,OAAO,EAAE;AAChD,YAAM,KAAK,YAAY,KAAK,QAAQ,IAAI,CAAA,OAAM,GAAG,GAAG,CAAC;AAErD,iBAAW,SAAS,QAAQ;AAC3B,YAAI,CAAC,MAAM,OAAO,IAAI;AACrB,gBAAM,KAAK,WAAW,KAAK,KAAK;AAAA,QACjC;AAAA,MACD;AACA,aAAO,CAAC,QAAQ,QAAQ,OAAO,SAAS,QAAQ,MAAM;AAAA,IACvD,CAAC;AAAA,EACF;AAAA,EAEA,MAAa,QAAQ,IAAY,QAAiC;AACjE,UAAMD,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO,QACN,GAAG;AAAA,QACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,QACA;AAAA,MAAA;AAAA,MAEF,uCAAuB,KAAA;AAAA,MACvB,gCAAgB,KAAA;AAAA,IAAK,CACrB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,YAAY,IAAY,QAAiC;AACrE,UAAMA,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,YAAY,IAAY,MAAiC;AACtE,UAAM,0BAAU,KAAA;AAChB,UAAMA,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,aAAa;AAAA,MACb,mBAAmB;AAAA,IAAA,CACnB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,KAAK,IAAI,CAAAC,QAAMA,IAAG,EAAE,CAAC,CAAC,EAC9C,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,WAAW,IAAY,OAAoC;AACxE,UAAM,aAAa,MAAM,IAAI,cAAc;AAC3C,UAAM,0BAAU,KAAA;AAChB,UAAM,kCAAkB,KAAA;AACxB,gBAAY;AAAA,MACX,YAAY,QAAA,KAAa,MAAM,QAAQ,2BAA2B,8BAA8B,KAAK,IAAI,GAAG,MAAM,IAAI,WAAW;AAAA,IAAA;AAElI,UAAMD,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,mBAAmB;AAAA,MACnB,aAAa;AAAA,MACb,KAAK,KAAK,UAAU,CAAC,GAAG,MAAM,IAAI,KAAK,MAAM,MAAM,CAAC;AAAA,MACpD,GAAI,cAAc,MAAM,QAAQ,eAAe,wBAC5C;AAAA,QACD,OAAO;AAAA,QACP,YAAY;AAAA,MAAA,IAEX;AAAA,QACD,OAAO;AAAA,MAAA;AAAA,IACR,CACD,EACA,MAAM,EAAE,IAAI,MAAM,IAAI,GAAA,CAAI,EAC1B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,0BAA0B,IAAY,YAAoB,gBAAwB;AAC/F,UAAMA,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB;AAAA,MAAM,QACN,GAAG;AAAA,QAAG,CAAAC,QACLA,IACE;AAAA,UAAI,CAAAA,QACJA,IACE,GAAG,SAAS,CAAC,YAAY,WAAW,YAAY,CAAC,EACjD,QAAQ,cAAc,SAAS,KAAK,KAAK,EACzC,QAAQ,UAAU,SAAS,IAAI,UAAU;AAAA,QAAA,EAE3C;AAAA,UAAI,CAAAA,QACJA,IACE,QAAQ,MAAM,SAAS,IAAI,cAAc;AAAA,QAAA;AAAA,MAC5C;AAAA,IACF,EAEA,OAAiB;AAAA;AAAA,MAEjB,uCAAuB,KAAA;AAAA,MACvB,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,KAAK,CAAA,OAAM,GAAG,IAAI,mBAAmB,EAAE,IAAI,OAAO,cAAc,gBAAgB,UAAU,wBAAA,CAAyB;AAAA,MACnH,OAAO;AAAA,MACP,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,QAAQ,EAAE;AAAA,EACb;AACD;"}
+{"version":3,"file":"EventsRepository.js","sources":["../../../../../packages/engine-actions/src/dispatch/EventsRepository.ts"],"sourcesContent":["import { Client, Connection, Literal, LockModifier, LockType, Operator, SelectBuilder, UpdateBuilder } from '@contember/database'\nimport { EventRow, HandledEvent } from './types.js'\nimport { Actions } from '@contember/schema'\nimport { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification.js'\nimport { notify } from '../utils/notifyChannel.js'\nimport { Logger } from '@contember/logger'\n\nconst ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes\nconst DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds\nconst DEFAULT_MAX_ATTEMPTS = 10\nconst DEFAULT_BATCH_SIZE = 1\n\nexport class EventsRepository {\n\tpublic async fetchBatch(actions: Actions.Schema, db: Client, logger: Logger): Promise<FetchBatchResult> {\n\t\t// Events referencing a target that no longer exists in the schema are terminally failed and\n\t\t// skipped over; count them so the caller can report them as terminal failures.\n\t\tlet unknownTargetFailed = 0\n\t\twhile (true) {\n\t\t\tconst primaryEvent = (await this.fetchInternal(db, 1))[0]\n\t\t\tif (!primaryEvent) {\n\t\t\t\treturn { ok: false, backoffMs: await this.fetchBackOff(db), unknownTargetFailed }\n\t\t\t}\n\t\t\tconst target = actions.targets[primaryEvent.target]\n\t\t\tif (!target) {\n\t\t\t\tif (await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)) {\n\t\t\t\t\tunknownTargetFailed++\n\t\t\t\t\t// Never retried: the schema has to name the target again before such an event can be delivered.\n\t\t\t\t\tlogger.error('Action event failed permanently, its target is not in the schema', {\n\t\t\t\t\t\teventId: primaryEvent.id,\n\t\t\t\t\t\ttarget: primaryEvent.target,\n\t\t\t\t\t\ttrigger: primaryEvent.trigger,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tconst batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1\n\t\t\tconst batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []\n\t\t\treturn { ok: true, events: [primaryEvent, ...batch], target, unknownTargetFailed }\n\t\t}\n\t}\n\n\tprivate async fetchBackOff(db: Client): Promise<number | undefined> {\n\t\tconst result = await SelectBuilder.create<{ ms: number }>()\n\t\t\t.from('actions_event')\n\t\t\t.match(eventsToProcessStateSpecification)\n\t\t\t.select(it => it.raw('ceil(extract(epoch from visible_at - now()) * 1000)'), 'ms')\n\t\t\t.orderBy('visible_at', 'asc')\n\t\t\t.limit(1)\n\t\t\t.getResult(db)\n\n\t\treturn result.length ? Math.max(0, result[0].ms) : undefined\n\t}\n\n\tprivate async fetchInternal(db: Client, limit: number, targetName?: string): Promise<EventRow[]> {\n\t\tconst visibleAt = new Date()\n\t\tvisibleAt.setTime(visibleAt.getTime() + ACK_TIMEOUT_MS)\n\t\treturn await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.with('events', qb =>\n\t\t\t\tqb\n\t\t\t\t\t.select(it => it.raw('*'))\n\t\t\t\t\t.from('actions_event')\n\t\t\t\t\t.match(eventsToProcessSpecification)\n\t\t\t\t\t.where(targetName ? { target: targetName } : {})\n\t\t\t\t\t.limit(limit)\n\t\t\t\t\t.lock(LockType.forNoKeyUpdate, LockModifier.skipLocked))\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'processing',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: visibleAt,\n\t\t\t})\n\t\t\t.from(it => it.from('events').where(it => it.columnsEq(['events', 'id'], ['actions_event', 'id'])))\n\t\t\t.returning<EventRow>(new Literal('*'))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async persistProcessed(db: Client, events: HandledEvent[], logger: Logger): Promise<PersistProcessedResult> {\n\t\treturn await db.transaction(async trx => {\n\t\t\tawait trx.query(Connection.REPEATABLE_READ)\n\t\t\tconst succeed = events.filter(it => it.result.ok)\n\t\t\tawait this.markSucceed(trx, succeed.map(it => it.row))\n\n\t\t\tlet retried = 0\n\t\t\tlet failed = 0\n\t\t\tfor (const event of events) {\n\t\t\t\tif (!event.result.ok) {\n\t\t\t\t\tconst terminal = await this.markFailed(trx, event)\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\tfailed++\n\t\t\t\t\t\t// Nothing else reports a given-up event; the target is named, never described by URL or headers.\n\t\t\t\t\t\tlogger.error('Action event failed permanently, no attempts left', {\n\t\t\t\t\t\t\teventId: event.row.id,\n\t\t\t\t\t\t\ttarget: event.row.target,\n\t\t\t\t\t\t\ttrigger: event.row.trigger,\n\t\t\t\t\t\t\tattempts: event.row.num_retries + 1,\n\t\t\t\t\t\t\tcode: event.result.code,\n\t\t\t\t\t\t\terrorMessage: event.result.errorMessage,\n\t\t\t\t\t\t})\n\t\t\t\t\t} else {\n\t\t\t\t\t\tretried++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn { succeeded: succeed.length, retried, failed }\n\t\t})\n\t}\n\n\tpublic async requeue(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: it =>\n\t\t\t\t\tit.raw(\n\t\t\t\t\t\t`\n\t\t\t\t\tCASE \n\t\t\t\t\tWHEN num_retries = 0 THEN \n\t\t\t\t\t\t?::__SCHEMA__.actions_trigger_state \n\t\t\t\t\tELSE \n\t\t\t\t\t?::__SCHEMA__.actions_trigger_state END\n\t\t\t\t`,\n\t\t\t\t\t\t'created',\n\t\t\t\t\t\t'retrying',\n\t\t\t\t\t),\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tvisible_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tpublic async markStopped(db: Client, events: string[]): Promise<void> {\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'stopped',\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.where(it => it.in('id', events))\n\t\t\t.execute(db)\n\t}\n\n\tprivate async markSucceed(db: Client, rows: EventRow[]): Promise<void> {\n\t\tconst now = new Date()\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tstate: 'succeed',\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tresolved_at: now,\n\t\t\t\tlast_state_change: now,\n\t\t\t})\n\t\t\t.where(it => it.in('id', rows.map(it => it.id)))\n\t\t\t.execute(db)\n\t}\n\n\t/** Returns `true` when the event reached a terminal `failed` state, `false` when it will be retried. */\n\tprivate async markFailed(db: Client, event: HandledEvent): Promise<boolean> {\n\t\tconst numRetries = event.row.num_retries + 1\n\t\tconst now = new Date()\n\t\tconst nextAttempt = new Date()\n\t\tnextAttempt.setTime(\n\t\t\tnextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries),\n\t\t)\n\t\tconst terminal = numRetries >= (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)\n\t\tawait UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.values<EventRow>({\n\t\t\t\tlast_state_change: now,\n\t\t\t\tnum_retries: numRetries,\n\t\t\t\tlog: JSON.stringify([...event.row.log, event.result]) as any,\n\t\t\t\t...(terminal\n\t\t\t\t\t? {\n\t\t\t\t\t\tstate: 'failed',\n\t\t\t\t\t}\n\t\t\t\t\t: {\n\t\t\t\t\t\tstate: 'retrying',\n\t\t\t\t\t\tvisible_at: nextAttempt,\n\t\t\t\t\t}),\n\t\t\t})\n\t\t\t.where({ id: event.row.id })\n\t\t\t.execute(db)\n\t\treturn terminal\n\t}\n\n\tprivate async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string): Promise<boolean> {\n\t\tconst affectedRows = await UpdateBuilder.create()\n\t\t\t.table('actions_event')\n\t\t\t.where(it =>\n\t\t\t\tit.or(it =>\n\t\t\t\t\tit\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.in('state', ['processing'])\n\t\t\t\t\t\t\t\t.compare('target', Operator.eq, targetName)\n\t\t\t\t\t\t)\n\t\t\t\t\t\t.and(it =>\n\t\t\t\t\t\t\tit\n\t\t\t\t\t\t\t\t.compare('id', Operator.eq, primaryEventId)\n\t\t\t\t\t\t)\n\t\t\t\t)\n\t\t\t)\n\t\t\t.values<EventRow>({\n\t\t\t\t// intentionally not retrying\n\t\t\t\tlast_state_change: new Date(),\n\t\t\t\tnum_retries: it => it.raw('num_retries + 1'),\n\t\t\t\tlog: it => it.raw('log || ?::jsonb', { ok: false, errorMessage: `Event target ${targetName} not found, stopping.` }),\n\t\t\t\tstate: 'failed',\n\t\t\t\tresolved_at: new Date(),\n\t\t\t})\n\t\t\t.execute(db)\n\t\treturn affectedRows > 0\n\t}\n}\n\nexport type FetchBatchResult =\n\t| { ok: true; events: EventRow[]; target: Actions.AnyTarget; unknownTargetFailed: number }\n\t| { ok: false; backoffMs?: number; unknownTargetFailed: number }\n\nexport type PersistProcessedResult = {\n\t/** Events delivered successfully (terminal). */\n\tsucceeded: number\n\t/** Events whose delivery attempt failed and that will be retried later. */\n\tretried: number\n\t/** Events whose delivery attempt failed and that reached a terminal `failed` state. */\n\tfailed: number\n}\n"],"names":["UpdateBuilder","it"],"mappings":";;AAOA,MAAM,iBAAiB,MAAQ,KAAK;AACpC,MAAM,6BAA6B;AACnC,MAAM,uBAAuB;AAC7B,MAAM,qBAAqB;AAEpB,MAAM,iBAAiB;AAAA,EAC7B,MAAa,WAAW,SAAyB,IAAY,QAA2C;AAGvG,QAAI,sBAAsB;AAC1B,WAAO,MAAM;AACZ,YAAM,gBAAgB,MAAM,KAAK,cAAc,IAAI,CAAC,GAAG,CAAC;AACxD,UAAI,CAAC,cAAc;AAClB,eAAO,EAAE,IAAI,OAAO,WAAW,MAAM,KAAK,aAAa,EAAE,GAAG,oBAAA;AAAA,MAC7D;AACA,YAAM,SAAS,QAAQ,QAAQ,aAAa,MAAM;AAClD,UAAI,CAAC,QAAQ;AACZ,YAAI,MAAM,KAAK,0BAA0B,IAAI,aAAa,QAAQ,aAAa,EAAE,GAAG;AACnF;AAEA,iBAAO,MAAM,oEAAoE;AAAA,YAChF,SAAS,aAAa;AAAA,YACtB,QAAQ,aAAa;AAAA,YACrB,SAAS,aAAa;AAAA,UAAA,CACtB;AAAA,QACF;AACA;AAAA,MACD;AACA,YAAM,aAAa,OAAO,aAAa,sBAAsB;AAC7D,YAAM,QAAQ,YAAY,IAAI,MAAM,KAAK,cAAc,IAAI,SAAS,IAAI,CAAA;AACxE,aAAO,EAAE,IAAI,MAAM,QAAQ,CAAC,cAAc,GAAG,KAAK,GAAG,QAAQ,oBAAA;AAAA,IAC9D;AAAA,EACD;AAAA,EAEA,MAAc,aAAa,IAAyC;AACnE,UAAM,SAAS,MAAM,cAAc,OAAA,EACjC,KAAK,eAAe,EACpB,MAAM,iCAAiC,EACvC,OAAO,CAAA,OAAM,GAAG,IAAI,qDAAqD,GAAG,IAAI,EAChF,QAAQ,cAAc,KAAK,EAC3B,MAAM,CAAC,EACP,UAAU,EAAE;AAEd,WAAO,OAAO,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI;AAAA,EACpD;AAAA,EAEA,MAAc,cAAc,IAAY,OAAe,YAA0C;AAChG,UAAM,gCAAgB,KAAA;AACtB,cAAU,QAAQ,UAAU,QAAA,IAAY,cAAc;AACtD,WAAO,MAAMA,gBAAc,OAAA,EACzB,MAAM,eAAe,EACrB,KAAK,UAAU,QACf,GACE,OAAO,QAAM,GAAG,IAAI,GAAG,CAAC,EACxB,KAAK,eAAe,EACpB,MAAM,4BAA4B,EAClC,MAAM,aAAa,EAAE,QAAQ,WAAA,IAAe,EAAE,EAC9C,MAAM,KAAK,EACX,KAAK,SAAS,gBAAgB,aAAa,UAAU,CAAC,EACxD,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,YAAY;AAAA,IAAA,CACZ,EACA,KAAK,CAAA,OAAM,GAAG,KAAK,QAAQ,EAAE,MAAM,CAAAC,QAAMA,IAAG,UAAU,CAAC,UAAU,IAAI,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,EACjG,UAAoB,IAAI,QAAQ,GAAG,CAAC,EACpC,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,iBAAiB,IAAY,QAAwB,QAAiD;AAClH,WAAO,MAAM,GAAG,YAAY,OAAM,QAAO;AACxC,YAAM,IAAI,MAAM,WAAW,eAAe;AAC1C,YAAM,UAAU,OAAO,OAAO,CAAA,OAAM,GAAG,OAAO,EAAE;AAChD,YAAM,KAAK,YAAY,KAAK,QAAQ,IAAI,CAAA,OAAM,GAAG,GAAG,CAAC;AAErD,UAAI,UAAU;AACd,UAAI,SAAS;AACb,iBAAW,SAAS,QAAQ;AAC3B,YAAI,CAAC,MAAM,OAAO,IAAI;AACrB,gBAAM,WAAW,MAAM,KAAK,WAAW,KAAK,KAAK;AACjD,cAAI,UAAU;AACb;AAEA,mBAAO,MAAM,qDAAqD;AAAA,cACjE,SAAS,MAAM,IAAI;AAAA,cACnB,QAAQ,MAAM,IAAI;AAAA,cAClB,SAAS,MAAM,IAAI;AAAA,cACnB,UAAU,MAAM,IAAI,cAAc;AAAA,cAClC,MAAM,MAAM,OAAO;AAAA,cACnB,cAAc,MAAM,OAAO;AAAA,YAAA,CAC3B;AAAA,UACF,OAAO;AACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AACA,aAAO,EAAE,WAAW,QAAQ,QAAQ,SAAS,OAAA;AAAA,IAC9C,CAAC;AAAA,EACF;AAAA,EAEA,MAAa,QAAQ,IAAY,QAAiC;AACjE,UAAMD,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO,QACN,GAAG;AAAA,QACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAOA;AAAA,QACA;AAAA,MAAA;AAAA,MAEF,uCAAuB,KAAA;AAAA,MACvB,gCAAgB,KAAA;AAAA,IAAK,CACrB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAa,YAAY,IAAY,QAAiC;AACrE,UAAMA,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,uCAAuB,KAAA;AAAA,MACvB,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,MAAM,CAAC,EAC/B,QAAQ,EAAE;AAAA,EACb;AAAA,EAEA,MAAc,YAAY,IAAY,MAAiC;AACtE,UAAM,0BAAU,KAAA;AAChB,UAAMA,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,OAAO;AAAA,MACP,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,aAAa;AAAA,MACb,mBAAmB;AAAA,IAAA,CACnB,EACA,MAAM,CAAA,OAAM,GAAG,GAAG,MAAM,KAAK,IAAI,CAAAC,QAAMA,IAAG,EAAE,CAAC,CAAC,EAC9C,QAAQ,EAAE;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,WAAW,IAAY,OAAuC;AAC3E,UAAM,aAAa,MAAM,IAAI,cAAc;AAC3C,UAAM,0BAAU,KAAA;AAChB,UAAM,kCAAkB,KAAA;AACxB,gBAAY;AAAA,MACX,YAAY,QAAA,KAAa,MAAM,QAAQ,2BAA2B,8BAA8B,KAAK,IAAI,GAAG,MAAM,IAAI,WAAW;AAAA,IAAA;AAElI,UAAM,WAAW,eAAe,MAAM,QAAQ,eAAe;AAC7D,UAAMD,gBAAc,OAAA,EAClB,MAAM,eAAe,EACrB,OAAiB;AAAA,MACjB,mBAAmB;AAAA,MACnB,aAAa;AAAA,MACb,KAAK,KAAK,UAAU,CAAC,GAAG,MAAM,IAAI,KAAK,MAAM,MAAM,CAAC;AAAA,MACpD,GAAI,WACD;AAAA,QACD,OAAO;AAAA,MAAA,IAEN;AAAA,QACD,OAAO;AAAA,QACP,YAAY;AAAA,MAAA;AAAA,IACb,CACD,EACA,MAAM,EAAE,IAAI,MAAM,IAAI,GAAA,CAAI,EAC1B,QAAQ,EAAE;AACZ,WAAO;AAAA,EACR;AAAA,EAEA,MAAc,0BAA0B,IAAY,YAAoB,gBAA0C;AACjH,UAAM,eAAe,MAAMA,gBAAc,SACvC,MAAM,eAAe,EACrB;AAAA,MAAM,QACN,GAAG;AAAA,QAAG,CAAAC,QACLA,IACE;AAAA,UAAI,CAAAA,QACJA,IACE,GAAG,SAAS,CAAC,YAAY,CAAC,EAC1B,QAAQ,UAAU,SAAS,IAAI,UAAU;AAAA,QAAA,EAE3C;AAAA,UAAI,CAAAA,QACJA,IACE,QAAQ,MAAM,SAAS,IAAI,cAAc;AAAA,QAAA;AAAA,MAC5C;AAAA,IACF,EAEA,OAAiB;AAAA;AAAA,MAEjB,uCAAuB,KAAA;AAAA,MACvB,aAAa,CAAA,OAAM,GAAG,IAAI,iBAAiB;AAAA,MAC3C,KAAK,CAAA,OAAM,GAAG,IAAI,mBAAmB,EAAE,IAAI,OAAO,cAAc,gBAAgB,UAAU,wBAAA,CAAyB;AAAA,MACnH,OAAO;AAAA,MACP,iCAAiB,KAAA;AAAA,IAAK,CACtB,EACA,QAAQ,EAAE;AACZ,WAAO,eAAe;AAAA,EACvB;AACD;"}