@checkstack/automation-backend 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +544 -0
  2. package/drizzle/0003_sparkling_xorn.sql +17 -0
  3. package/drizzle/0004_cultured_spyke.sql +2 -0
  4. package/drizzle/0005_classy_the_hand.sql +19 -0
  5. package/drizzle/0006_burly_wallop.sql +10 -0
  6. package/drizzle/0007_nappy_jackal.sql +1 -0
  7. package/drizzle/0008_remove_seeded_auto_incident_automations.sql +13 -0
  8. package/drizzle/0009_steady_liz_osborn.sql +12 -0
  9. package/drizzle/0010_chunky_changeling.sql +2 -0
  10. package/drizzle/meta/0003_snapshot.json +1007 -0
  11. package/drizzle/meta/0004_snapshot.json +1028 -0
  12. package/drizzle/meta/0005_snapshot.json +1164 -0
  13. package/drizzle/meta/0006_snapshot.json +1261 -0
  14. package/drizzle/meta/0007_snapshot.json +1215 -0
  15. package/drizzle/meta/0008_snapshot.json +1215 -0
  16. package/drizzle/meta/0009_snapshot.json +1328 -0
  17. package/drizzle/meta/0010_snapshot.json +1349 -0
  18. package/drizzle/meta/_journal.json +56 -0
  19. package/package.json +23 -12
  20. package/src/action-types.ts +23 -0
  21. package/src/artifact-store.ts +16 -1
  22. package/src/automation-store.test.ts +143 -0
  23. package/src/automation-store.ts +30 -8
  24. package/src/builtin-triggers.test.ts +77 -74
  25. package/src/builtin-triggers.ts +105 -108
  26. package/src/dispatch/action-kind.ts +2 -0
  27. package/src/dispatch/assemble-get-service.ts +31 -0
  28. package/src/dispatch/cancel-resurrect.test.ts +147 -0
  29. package/src/dispatch/concurrency-race.test.ts +255 -0
  30. package/src/dispatch/concurrency-scope.test.ts +166 -0
  31. package/src/dispatch/condition.ts +24 -5
  32. package/src/dispatch/dwell-queue.ts +65 -0
  33. package/src/dispatch/dwell-store.ts +154 -0
  34. package/src/dispatch/dwell.it.test.ts +142 -0
  35. package/src/dispatch/dwell.test.ts +799 -0
  36. package/src/dispatch/dwell.ts +257 -0
  37. package/src/dispatch/engine.test.ts +189 -2
  38. package/src/dispatch/engine.ts +555 -9
  39. package/src/dispatch/entity-scope.test.ts +176 -0
  40. package/src/dispatch/get-service-wiring.test.ts +318 -0
  41. package/src/dispatch/numeric.test.ts +71 -0
  42. package/src/dispatch/numeric.ts +96 -0
  43. package/src/dispatch/render.test.ts +34 -0
  44. package/src/dispatch/render.ts +31 -11
  45. package/src/dispatch/reseed-run-secrets.ts +230 -0
  46. package/src/dispatch/run-secret-registry.test.ts +189 -0
  47. package/src/dispatch/run-secret-registry.ts +247 -0
  48. package/src/dispatch/run-state-masking.test.ts +376 -0
  49. package/src/dispatch/run-state-store.ts +95 -38
  50. package/src/dispatch/run-state.ts +226 -59
  51. package/src/dispatch/scope-artifact-masking.test.ts +138 -0
  52. package/src/dispatch/secret-ref-ids.test.ts +19 -0
  53. package/src/dispatch/secret-ref-ids.ts +17 -0
  54. package/src/dispatch/snapshots.test.ts +86 -0
  55. package/src/dispatch/snapshots.ts +79 -0
  56. package/src/dispatch/stage1-router.test.ts +324 -0
  57. package/src/dispatch/stage1-router.ts +152 -0
  58. package/src/dispatch/stage1.it.test.ts +84 -0
  59. package/src/dispatch/stage2-dispatch.test.ts +285 -0
  60. package/src/dispatch/stage2-dispatch.ts +207 -0
  61. package/src/dispatch/stage2-stalled.it.test.ts +132 -0
  62. package/src/dispatch/stalled-sweeper.test.ts +197 -0
  63. package/src/dispatch/stalled-sweeper.ts +112 -5
  64. package/src/dispatch/state-scope.test.ts +234 -0
  65. package/src/dispatch/state-scope.ts +322 -0
  66. package/src/dispatch/structured-conditions.test.ts +246 -0
  67. package/src/dispatch/structured-conditions.ts +146 -0
  68. package/src/dispatch/test-fixtures.ts +306 -38
  69. package/src/dispatch/trigger-fanin.test.ts +111 -0
  70. package/src/dispatch/trigger-subscriber.ts +316 -14
  71. package/src/dispatch/types.ts +263 -8
  72. package/src/dispatch/wait-timeout-queue.ts +89 -0
  73. package/src/dispatch/wait-until-entity-wake.test.ts +544 -0
  74. package/src/dispatch/wait-until.test.ts +540 -0
  75. package/src/dispatch/wake-refs.test.ts +158 -0
  76. package/src/dispatch/wake-refs.ts +348 -0
  77. package/src/dispatch/window-gate.test.ts +513 -0
  78. package/src/dispatch/window-store.test.ts +162 -0
  79. package/src/dispatch/window-store.ts +102 -0
  80. package/src/entity/change-derivers.test.ts +148 -0
  81. package/src/entity/change-derivers.ts +143 -0
  82. package/src/entity/change-emitter.test.ts +66 -0
  83. package/src/entity/change-emitter.ts +76 -0
  84. package/src/entity/create-handle.ts +344 -0
  85. package/src/entity/cross-pod-read-consistency.it.test.ts +281 -0
  86. package/src/entity/define-entity.ts +157 -0
  87. package/src/entity/diff.test.ts +57 -0
  88. package/src/entity/diff.ts +54 -0
  89. package/src/entity/entity-store.test.ts +30 -0
  90. package/src/entity/entity-store.ts +171 -0
  91. package/src/entity/extension-point.ts +56 -0
  92. package/src/entity/fake-entity-store.ts +130 -0
  93. package/src/entity/hook.ts +19 -0
  94. package/src/entity/index.ts +50 -0
  95. package/src/entity/mutate-handle.test.ts +517 -0
  96. package/src/entity/on-entity-changed.test.ts +189 -0
  97. package/src/entity/on-entity-changed.ts +214 -0
  98. package/src/entity/registry.test.ts +181 -0
  99. package/src/entity/registry.ts +200 -0
  100. package/src/entity/stable-stringify.test.ts +55 -0
  101. package/src/entity/stable-stringify.ts +49 -0
  102. package/src/entity/wake-index.it.test.ts +251 -0
  103. package/src/entity/with-entity-write.test.ts +100 -0
  104. package/src/entity/with-entity-write.ts +69 -0
  105. package/src/entity-driven-trigger.ts +46 -0
  106. package/src/extension-points.ts +35 -0
  107. package/src/gitops-docs.test.ts +215 -0
  108. package/src/gitops-docs.ts +151 -0
  109. package/src/gitops-kinds.test.ts +174 -0
  110. package/src/gitops-kinds.ts +137 -0
  111. package/src/index.ts +355 -11
  112. package/src/migration/flapping-to-window.test.ts +123 -0
  113. package/src/migration/flapping-to-window.ts +205 -0
  114. package/src/router.test.ts +182 -1
  115. package/src/router.ts +73 -2
  116. package/src/schema.ts +236 -3
  117. package/src/script-test-replay.test.ts +88 -0
  118. package/src/script-test-replay.ts +100 -0
  119. package/src/script-test-shell-env.test.ts +41 -0
  120. package/src/script-test-shell-env.ts +89 -0
  121. package/src/script-test.test.ts +386 -0
  122. package/src/script-test.ts +258 -0
  123. package/src/trigger-registry.ts +2 -0
  124. package/src/validate-definition.test.ts +1 -0
  125. package/tsconfig.json +24 -0
package/src/index.ts CHANGED
@@ -1,6 +1,8 @@
1
1
  import {
2
2
  createBackendPlugin,
3
3
  coreServices,
4
+ withXactLock,
5
+ type SafeDatabase,
4
6
  } from "@checkstack/backend-api";
5
7
  import {
6
8
  automationAccess,
@@ -12,7 +14,19 @@ import {
12
14
  import { resolveRoute, extractErrorMessage } from "@checkstack/common";
13
15
  import type { PluginMetadata } from "@checkstack/common";
14
16
  import { registerSearchProvider } from "@checkstack/command-backend";
15
- import { createDefaultFilterRegistry } from "@checkstack/template-engine";
17
+ import {
18
+ createDefaultFilterRegistry,
19
+ type FilterRegistry,
20
+ } from "@checkstack/template-engine";
21
+ import { HealthCheckApi } from "@checkstack/healthcheck-common";
22
+ import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
23
+ import { CHECKSTACK_API_VERSION } from "@checkstack/gitops-common";
24
+ import {
25
+ reconcileAutomation,
26
+ deleteAutomationEntity,
27
+ } from "./gitops-kinds";
28
+ import { registerAutomationGitOpsDocumentation } from "./gitops-docs";
29
+ import { AutomationDefinitionSchema } from "@checkstack/automation-common";
16
30
 
17
31
  import type {
18
32
  ActionDefinition,
@@ -29,12 +43,36 @@ import { createArtifactStore } from "./artifact-store";
29
43
  import { createAutomationStore } from "./automation-store";
30
44
  import { createAutomationRouter } from "./router";
31
45
  import { runWebhookSubscriptionMigration } from "./migration/from-webhook-subscriptions";
46
+ import { runFlappingAutomationMigration } from "./migration/flapping-to-window";
32
47
  import {
33
48
  startDelayQueueConsumer,
34
49
  type DelayQueueConsumer,
35
50
  } from "./dispatch/delay-queue";
51
+ import {
52
+ startDwellQueueConsumer,
53
+ type DwellQueueConsumer,
54
+ } from "./dispatch/dwell-queue";
55
+ import {
56
+ startWaitTimeoutQueueConsumer,
57
+ type WaitTimeoutQueueConsumer,
58
+ } from "./dispatch/wait-timeout-queue";
59
+ import {
60
+ startDispatchQueueConsumer,
61
+ type DispatchQueueConsumer,
62
+ } from "./dispatch/stage2-dispatch";
63
+ import {
64
+ startStage1Router,
65
+ type Stage1Router,
66
+ } from "./dispatch/stage1-router";
67
+ import { createDwellStore } from "./dispatch/dwell-store";
68
+ import { createWindowStore } from "./dispatch/window-store";
36
69
  import { createRunStore } from "./dispatch/run-state";
37
70
  import { createRunStateStore } from "./dispatch/run-state-store";
71
+ import { createRunSecretRegistry } from "./dispatch/run-secret-registry";
72
+ import {
73
+ SECRET_RESOLVER_REF_ID,
74
+ CONNECTION_STORE_REF_ID,
75
+ } from "./dispatch/secret-ref-ids";
38
76
  import {
39
77
  startStalledSweeper,
40
78
  type StalledSweeper,
@@ -44,10 +82,12 @@ import {
44
82
  type TriggerSubscriptions,
45
83
  } from "./dispatch/trigger-subscriber";
46
84
  import type { DispatchDeps } from "./dispatch/types";
85
+ import { assembleDispatchGetService } from "./dispatch/assemble-get-service";
47
86
  import {
48
87
  automationActionExtensionPoint,
49
88
  automationArtifactStoreRef,
50
89
  automationArtifactTypeExtensionPoint,
90
+ automationFilterExtensionPoint,
51
91
  automationRegistriesRef,
52
92
  automationTriggerExtensionPoint,
53
93
  } from "./extension-points";
@@ -55,6 +95,19 @@ import {
55
95
  registerBuiltinTriggerConsumer,
56
96
  registerBuiltinTriggers,
57
97
  } from "./builtin-triggers";
98
+ import {
99
+ createChangeDeriverRegistry,
100
+ createChangeEmitter,
101
+ createEntityChangedSubscriptions,
102
+ createEntityRegistry,
103
+ createEntityStore,
104
+ entityExtensionPoint,
105
+ type ChangeDeriverRegistry,
106
+ type ChangeEmitter,
107
+ type EntityChangedSubscriptions,
108
+ type EntityRegistry,
109
+ } from "./entity";
110
+ import { ENTITY_CHANGED_HOOK } from "./entity/hook";
58
111
  import {
59
112
  createNotifyUserAction,
60
113
  logAction,
@@ -73,21 +126,75 @@ interface EnvStash {
73
126
  artifactTypeRegistry: ArtifactTypeRegistry;
74
127
  dispatchDeps: DispatchDeps;
75
128
  automationStore: ReturnType<typeof createAutomationStore>;
129
+ entityRegistry: EntityRegistry;
130
+ entityChangeEmitter: ChangeEmitter;
131
+ entityChangedSubscriptions: EntityChangedSubscriptions;
132
+ changeDerivers: ChangeDeriverRegistry;
76
133
  triggerSubscriptions?: TriggerSubscriptions;
77
134
  stalledSweeper?: StalledSweeper;
78
135
  delayConsumer?: DelayQueueConsumer;
136
+ dwellConsumer?: DwellQueueConsumer;
137
+ waitTimeoutConsumer?: WaitTimeoutQueueConsumer;
138
+ dispatchConsumer?: DispatchQueueConsumer;
139
+ stage1Router?: Stage1Router;
79
140
  }
80
141
 
81
142
  export default createBackendPlugin({
82
143
  metadata: pluginMetadata,
83
144
 
84
145
  register(env) {
146
+ // Mutable DB ref — populated in init(), consumed by the GitOps
147
+ // reconcile/delete closures (only called during sync, after init).
148
+ let gitopsDb: SafeDatabase<typeof schema> | undefined;
149
+
85
150
  const triggerRegistry = createTriggerRegistry();
86
151
  const actionRegistry = createActionRegistry();
87
152
  const artifactTypeRegistry = createArtifactTypeRegistry();
153
+ // Shared filter registry — seeded with the built-in defaults (incl.
154
+ // the Wave-2 duration helpers) and extended by plugins via the
155
+ // filter extension point in Phase 1. The dispatch engine reads from
156
+ // this same instance, so plugin filters are live by `init()`.
157
+ const filterRegistry: FilterRegistry = createDefaultFilterRegistry();
158
+
159
+ // Run-scoped secret registry — created here in `register()` (it has no
160
+ // deps) so the SAME instance backs both the dispatch masking choke
161
+ // points (wired in `init()`) and the entity-store masking choke point
162
+ // (run-originated entity writes mask through this in the handle).
163
+ const secretRegistry = createRunSecretRegistry();
164
+
165
+ // Entity state machine (reactive automation engine §4). The change
166
+ // emitter buffers until `emitHook` is wired in `afterPluginsReady`
167
+ // (§3.7); the registry exposes `defineEntity` / `declareNonReactiveState`
168
+ // through the extension point below, callable from other plugins'
169
+ // `register`/`init`. The DB-backed store is bound in `init()` (after
170
+ // migrations run).
171
+ const entityChangeEmitter = createChangeEmitter();
172
+ const entityRegistry = createEntityRegistry({
173
+ secretRegistry,
174
+ emitter: entityChangeEmitter,
175
+ });
176
+
177
+ // Reactive dispatch pipeline (reactive automation engine §7). The
178
+ // change-deriver registry maps a kind's change → trigger event id(s)
179
+ // for Stage-1 routing; per-domain derivers are registered in Phase 4.
180
+ // The entity-changed subscription service rides ENTITY_CHANGED (filtered
181
+ // by kind) so other plugins can react without touching the internal hook
182
+ // (§6.1). Both buffer registrations made before afterPluginsReady wires
183
+ // the real `onHook`.
184
+ const changeDerivers = createChangeDeriverRegistry();
185
+ const entityChangedSubscriptions = createEntityChangedSubscriptions();
88
186
 
89
187
  env.registerAccessRules(automationAccessRules);
90
188
 
189
+ // Phase 1: register the entity extension point so other plugins can
190
+ // resolve it and call `defineEntity` during their own register/init.
191
+ env.registerExtensionPoint(entityExtensionPoint, {
192
+ defineEntity: entityRegistry.defineEntity,
193
+ declareNonReactiveState: entityRegistry.declareNonReactiveState,
194
+ onEntityChanged: entityChangedSubscriptions.onEntityChanged,
195
+ registerChangeDeriver: (input) => changeDerivers.register(input),
196
+ });
197
+
91
198
  env.registerExtensionPoint(automationTriggerExtensionPoint, {
92
199
  registerTrigger: <TPayload, TConfig = void>(
93
200
  definition: TriggerDefinition<TPayload, TConfig>,
@@ -124,6 +231,64 @@ export default createBackendPlugin({
124
231
  },
125
232
  });
126
233
 
234
+ // Filters registered by plugins in Phase 1 are collected here and
235
+ // applied (with collision-warning) in `init()` where a logger is
236
+ // available. Collecting first keeps `register()` logger-free.
237
+ const pendingFilters: Array<{
238
+ name: string;
239
+ filter: Parameters<FilterRegistry["register"]>[1];
240
+ pluginId: string;
241
+ }> = [];
242
+ env.registerExtensionPoint(automationFilterExtensionPoint, {
243
+ registerFilter: (definition, metadata) => {
244
+ pendingFilters.push({
245
+ name: definition.name,
246
+ filter: definition.filter,
247
+ pluginId: metadata.pluginId,
248
+ });
249
+ },
250
+ });
251
+
252
+ // GitOps `Automation` kind. The DB isn't available until init(), so a
253
+ // mutable ref is populated there and read by the reconcile closures
254
+ // (only invoked during sync, well after init) — mirrors catalog-backend.
255
+ const kindRegistry = env.getExtensionPoint(entityKindExtensionPoint);
256
+ kindRegistry.registerKind({
257
+ apiVersion: CHECKSTACK_API_VERSION,
258
+ kind: "Automation",
259
+ specSchema: AutomationDefinitionSchema,
260
+ reconcile: async ({ entity, existingEntityId, context }) => {
261
+ if (!gitopsDb) throw new Error("Automation database not initialized");
262
+ return reconcileAutomation(gitopsDb, {
263
+ entity,
264
+ existingEntityId,
265
+ logger: context.logger,
266
+ });
267
+ },
268
+ delete: async ({ entityId, context }) => {
269
+ if (!gitopsDb) throw new Error("Automation database not initialized");
270
+ await deleteAutomationEntity(gitopsDb, {
271
+ entityId,
272
+ logger: context.logger,
273
+ });
274
+ },
275
+ });
276
+
277
+ // Register the GitOps spec-schema documentation PROVIDER for the
278
+ // `Automation` kind HERE in register() (not afterPluginsReady). It is a
279
+ // LAZY provider: it re-reads the trigger/action registries on every
280
+ // kind-browser query (`describeKinds()`), so it needs no populated
281
+ // registries at registration time — registering it early, before any
282
+ // init / afterPluginsReady ordering, guarantees it is always present.
283
+ // Surfaces each trigger's / provider action's config schema in the Kind
284
+ // Registry, conditioned on the chosen `triggers[].event` /
285
+ // `actions[].action`.
286
+ registerAutomationGitOpsDocumentation({
287
+ kindRegistry,
288
+ triggerRegistry,
289
+ actionRegistry,
290
+ });
291
+
127
292
  env.registerInit({
128
293
  schema,
129
294
  deps: {
@@ -132,6 +297,7 @@ export default createBackendPlugin({
132
297
  rpcClient: coreServices.rpcClient,
133
298
  queueManager: coreServices.queueManager,
134
299
  signalService: coreServices.signalService,
300
+ advisoryLock: coreServices.advisoryLock,
135
301
  },
136
302
  init: async ({
137
303
  logger,
@@ -140,14 +306,37 @@ export default createBackendPlugin({
140
306
  rpcClient,
141
307
  queueManager,
142
308
  signalService,
309
+ advisoryLock,
143
310
  }) => {
144
311
  logger.debug("⚙️ Initializing Automation Backend...");
145
312
 
146
- const artifactStore = createArtifactStore(database);
147
- const runStore = createRunStore(database);
148
- const runStateStore = createRunStateStore(database);
313
+ // Populate the mutable DB ref the GitOps reconcile closures read.
314
+ gitopsDb = database as SafeDatabase<typeof schema>;
315
+
316
+ // Run-scoped secret registry: created in `register()` (the SAME
317
+ // instance) so it accumulates every secret value resolved during a
318
+ // run and every persistence choke point (run store step/run output,
319
+ // run-state scope snapshot, artifact data, AND run-originated entity
320
+ // writes) masks before write.
321
+ const artifactStore = createArtifactStore(database, secretRegistry);
322
+ const runStore = createRunStore(database, logger, secretRegistry);
323
+ const runStateStore = createRunStateStore(
324
+ database,
325
+ advisoryLock,
326
+ secretRegistry,
327
+ );
328
+ const dwellStore = createDwellStore(database);
329
+ const windowStore = createWindowStore(database);
149
330
  const automationStore = createAutomationStore(database);
150
331
 
332
+ // Bind the DB-backed transition store to the registry (the extension
333
+ // point impl registered in `register()` forwards through it). Model B:
334
+ // the transition store owns the tx + `entity_transitions` log for
335
+ // EVERY kind. Bound here in `init()` — after migrations have run — so
336
+ // the table exists.
337
+ const entityStore = createEntityStore(database);
338
+ entityRegistry.setStore({ store: entityStore });
339
+
151
340
  env.registerService(automationArtifactStoreRef, artifactStore);
152
341
  env.registerService(automationRegistriesRef, {
153
342
  triggers: triggerRegistry,
@@ -188,9 +377,21 @@ export default createBackendPlugin({
188
377
  );
189
378
  await registerBuiltinTriggerConsumer({ queueManager, logger });
190
379
 
380
+ // Apply plugin-contributed filters collected in register(),
381
+ // skipping any that would shadow a built-in (warn, don't clobber).
382
+ for (const pf of pendingFilters) {
383
+ if (filterRegistry.has(pf.name)) {
384
+ logger.warn(
385
+ `Plugin ${pf.pluginId} tried to register filter "${pf.name}" which already exists; skipping.`,
386
+ );
387
+ continue;
388
+ }
389
+ filterRegistry.register(pf.name, pf.filter);
390
+ }
391
+
191
392
  const dispatchDeps: DispatchDeps = {
192
393
  logger,
193
- filters: createDefaultFilterRegistry(),
394
+ filters: filterRegistry,
194
395
  registries: {
195
396
  triggers: triggerRegistry,
196
397
  actions: actionRegistry,
@@ -199,12 +400,40 @@ export default createBackendPlugin({
199
400
  artifactStore,
200
401
  runStore,
201
402
  runStateStore,
403
+ dwellStore,
404
+ windowStore,
202
405
  queueManager,
203
- getService: async () => {
204
- throw new Error(
205
- "getService not yet wired — automation dispatch invoked too early",
206
- );
207
- },
406
+ // Sensing-layer scope pre-resolution reads live health state
407
+ // through this client. forPlugin is lazy; the actual RPC only
408
+ // fires at evaluation time.
409
+ healthCheckClient: rpcClient.forPlugin(HealthCheckApi),
410
+ // Kind-agnostic entity resolver for reactive `wait_until` wake
411
+ // re-evaluation (Model B): the registry routes each kind to its
412
+ // plugin `read` accessor. Unknown kinds
413
+ // yield `undefined` (enrichment leaves them unresolved, fail-open).
414
+ // This is what lets a wait on `state.<kind>.<id>` (incident, slo, …)
415
+ // re-evaluate correctly when that kind changes (not just health).
416
+ entityResolverFor: (kind) => entityRegistry.entityResolverFor(kind),
417
+ // Registry-backed resolution of provider-action deps (connection
418
+ // store, secret resolver, ...) at execute time. Safe here because
419
+ // dispatch only runs from afterPluginsReady onward, by which point
420
+ // every service is registered. `env.getService` resolves through
421
+ // the real ServiceRegistry and throws clearly on a missing ref.
422
+ getService: assembleDispatchGetService({
423
+ envGetService: env.getService,
424
+ }),
425
+ // Run-wide secret masking: the engine wraps each run's getService
426
+ // to register resolved secrets here, and the run store masks step
427
+ // / run output before persistence.
428
+ secretRegistry,
429
+ secretResolverRefId: SECRET_RESOLVER_REF_ID,
430
+ connectionStoreRefId: CONNECTION_STORE_REF_ID,
431
+ // Serialize the concurrency-mode check-then-create with a
432
+ // transaction-scoped advisory lock (blocks until granted,
433
+ // auto-releases at COMMIT) so racing fires can't double-run a
434
+ // single-mode automation.
435
+ withConcurrencyLock: <T>(key: string, fn: () => Promise<T>) =>
436
+ withXactLock({ db: database, key, fn: () => fn() }),
208
437
  };
209
438
 
210
439
  const stash = env as unknown as EnvStash;
@@ -213,6 +442,10 @@ export default createBackendPlugin({
213
442
  stash.artifactTypeRegistry = artifactTypeRegistry;
214
443
  stash.dispatchDeps = dispatchDeps;
215
444
  stash.automationStore = automationStore;
445
+ stash.entityRegistry = entityRegistry;
446
+ stash.entityChangeEmitter = entityChangeEmitter;
447
+ stash.entityChangedSubscriptions = entityChangedSubscriptions;
448
+ stash.changeDerivers = changeDerivers;
216
449
 
217
450
  const router = createAutomationRouter({
218
451
  db: database,
@@ -260,19 +493,43 @@ export default createBackendPlugin({
260
493
  env.registerCleanup(async () => {
261
494
  const s = env as unknown as EnvStash;
262
495
  await s.triggerSubscriptions?.dispose();
496
+ await s.stage1Router?.dispose();
497
+ await s.entityChangedSubscriptions?.disposeAll();
263
498
  s.stalledSweeper?.stop();
264
499
  await s.delayConsumer?.stop();
500
+ await s.dwellConsumer?.stop();
501
+ await s.waitTimeoutConsumer?.stop();
502
+ await s.dispatchConsumer?.stop();
265
503
  });
266
504
 
267
505
  logger.debug("✅ Automation Backend initialized.");
268
506
  },
269
507
 
270
- afterPluginsReady: async ({ database, logger, onHook, rpcClient }) => {
508
+ afterPluginsReady: async ({
509
+ database,
510
+ logger,
511
+ onHook,
512
+ emitHook,
513
+ rpcClient,
514
+ }) => {
271
515
  const stash = env as unknown as EnvStash;
272
516
  const triggers = stash.triggerRegistry.getTriggers();
273
517
  const actions = stash.actionRegistry.getActions();
274
518
  const artifactTypes = stash.artifactTypeRegistry.getArtifactTypes();
275
519
 
520
+ // Wire the deferred entity-change emitter to the real `emitHook`
521
+ // (only injectable here — §3.7). Any change events buffered during
522
+ // the init / afterPluginsReady window are flushed in order now, so
523
+ // there is no silent no-emit gap.
524
+ await stash.entityChangeEmitter.wire((payload) =>
525
+ emitHook(ENTITY_CHANGED_HOOK, payload),
526
+ );
527
+
528
+ // Wire the public cross-plugin entity-change subscription service
529
+ // (§6.1). Subscriptions registered by other plugins during their
530
+ // register/init are bound to the real `onHook` now.
531
+ stash.entityChangedSubscriptions.wire({ onHook, logger });
532
+
276
533
  logger.debug(
277
534
  `⚙️ Registered ${triggers.length} automation triggers${
278
535
  triggers.length > 0
@@ -313,6 +570,46 @@ export default createBackendPlugin({
313
570
  logger,
314
571
  });
315
572
 
573
+ // `for:` dwell: register the consumer that fires when a dwell's
574
+ // scheduled job pops, re-confirming state before starting the run.
575
+ stash.dwellConsumer = await startDwellQueueConsumer({
576
+ deps: stash.dispatchDeps,
577
+ automationStore: stash.automationStore,
578
+ logger,
579
+ });
580
+
581
+ // Reactive `wait_until` timeout timer: the consumer that fires when
582
+ // a suspended wait's single deadline job pops, applying the
583
+ // continue/fail-on-timeout policy. Reactive waits are otherwise woken
584
+ // by Stage-1 routing on a relevant ENTITY_CHANGED (no polling).
585
+ stash.waitTimeoutConsumer = await startWaitTimeoutQueueConsumer({
586
+ deps: stash.dispatchDeps,
587
+ automationStore: stash.automationStore,
588
+ logger,
589
+ });
590
+
591
+ // Stage-2 dispatch fan-out: the consumer that runs each per-run
592
+ // dispatch job enqueued by Stage-1 routing (reason: trigger →
593
+ // dispatchTrigger; reason: wake → resume the suspended wait_until).
594
+ stash.dispatchConsumer = await startDispatchQueueConsumer({
595
+ deps: stash.dispatchDeps,
596
+ automationStore: stash.automationStore,
597
+ changeDerivers: stash.changeDerivers,
598
+ logger,
599
+ });
600
+
601
+ // Stage-1 routing: claim each ENTITY_CHANGED on the
602
+ // `automation-entity-route` work-queue (exactly one instance), do
603
+ // cheap indexed routing (wake-index intersection + trigger-event
604
+ // derivation), and enqueue Stage-2 jobs.
605
+ stash.stage1Router = await startStage1Router({
606
+ deps: stash.dispatchDeps,
607
+ automationStore: stash.automationStore,
608
+ changeDerivers: stash.changeDerivers,
609
+ onHook,
610
+ logger,
611
+ });
612
+
316
613
  // Restart safety + horizontal scaling: periodically scan for
317
614
  // runs whose heartbeat is older than the threshold and resume
318
615
  // them under an advisory lock.
@@ -340,6 +637,19 @@ export default createBackendPlugin({
340
637
  );
341
638
  }
342
639
 
640
+ // One-time migration: rewrite legacy `healthcheck.flapping_detected`
641
+ // triggers onto the generic windowed-count gate over
642
+ // `healthcheck.system_health_changed` (the flapping trigger + hook
643
+ // were removed). Idempotent — already-migrated / non-flapping rows are
644
+ // skipped — so it is safe to run on every boot.
645
+ try {
646
+ await runFlappingAutomationMigration({ db: database, logger });
647
+ } catch (error) {
648
+ logger.error(
649
+ `Flapping automation migration failed unexpectedly: ${extractErrorMessage(error, "unknown error")}`,
650
+ );
651
+ }
652
+
343
653
  logger.debug("✅ Automation Backend afterPluginsReady complete.");
344
654
  },
345
655
  });
@@ -356,6 +666,37 @@ export {
356
666
  automationArtifactStoreRef,
357
667
  } from "./extension-points";
358
668
 
669
+ // Entity state machine — the typed path to reactive state. The internal
670
+ // ENTITY_CHANGED hook is intentionally NOT re-exported (§6.1).
671
+ export { entityExtensionPoint } from "./entity";
672
+ export { withEntityWrite, withEntityRemove } from "./entity";
673
+ export type {
674
+ EntityExtensionPoint,
675
+ DefineEntity,
676
+ DefineEntityInput,
677
+ DeclareNonReactiveState,
678
+ DeclareNonReactiveStateInput,
679
+ EntityHandle,
680
+ EntityMutationOpts,
681
+ EntityRead,
682
+ MutateInput,
683
+ RemoveInput,
684
+ EntityTx,
685
+ EntityChangeDeriver,
686
+ EntityChangePayloadMapper,
687
+ RegisterChangeDeriver,
688
+ OnEntityChanged,
689
+ OnEntityChangedInput,
690
+ EntityChangedHandler,
691
+ EntityChangedDelivery,
692
+ EntityChangedUnsubscribe,
693
+ } from "./entity";
694
+
695
+ // The validated entity-change payload (Phase 4 derivers + cross-plugin
696
+ // consumers type against this). Re-exported from automation-common so a
697
+ // domain plugin needs only the automation-backend dependency.
698
+ export type { EntityChanged } from "@checkstack/automation-common";
699
+
359
700
  export type {
360
701
  TriggerDefinition,
361
702
  ActionDefinition,
@@ -371,9 +712,12 @@ export type {
371
712
  TriggerTeardown,
372
713
  } from "./action-types";
373
714
 
715
+ export { makeEntityDrivenTriggerSetup } from "./entity-driven-trigger";
716
+
374
717
  export type { ArtifactStore, PersistedArtifact } from "./artifact-store";
375
718
  export type { TriggerRegistry } from "./trigger-registry";
376
719
  export type { ActionRegistry } from "./action-registry";
377
720
  export type { ArtifactTypeRegistry } from "./artifact-type-registry";
378
721
  export type { AutomationRegistries } from "./extension-points";
379
722
  export type { AutomationStore } from "./automation-store";
723
+ export type { LoadedAutomation } from "./dispatch/types";
@@ -0,0 +1,123 @@
1
+ import { describe, expect, it } from "bun:test";
2
+
3
+ import {
4
+ DEFAULT_FLAPPING_COUNT,
5
+ DEFAULT_FLAPPING_MINUTES,
6
+ FLAPPING_FILTER,
7
+ HEALTH_CHANGED_EVENT,
8
+ LEGACY_FLAPPING_EVENT,
9
+ rewriteFlappingTriggers,
10
+ } from "./flapping-to-window";
11
+
12
+ function def(triggers: unknown[]): Record<string, unknown> {
13
+ return {
14
+ name: "Flapping",
15
+ triggers,
16
+ conditions: [],
17
+ actions: [{ action: "incident.create", config: {} }],
18
+ mode: "single",
19
+ max_runs: 1,
20
+ };
21
+ }
22
+
23
+ describe("rewriteFlappingTriggers", () => {
24
+ it("maps {transitions, windowMinutes} → window {count, minutes, refire: once}", () => {
25
+ const out = rewriteFlappingTriggers(
26
+ def([
27
+ {
28
+ id: "flap",
29
+ event: LEGACY_FLAPPING_EVENT,
30
+ config: { transitions: 5, windowMinutes: 15 },
31
+ },
32
+ ]),
33
+ );
34
+ expect(out.rewritten).toBe(1);
35
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
36
+ expect(triggers[0]).toEqual({
37
+ id: "flap",
38
+ event: HEALTH_CHANGED_EVENT,
39
+ filter: FLAPPING_FILTER,
40
+ window: { count: 5, minutes: 15, refire: "once" },
41
+ });
42
+ // config is dropped
43
+ expect("config" in triggers[0]!).toBe(false);
44
+ });
45
+
46
+ it("applies 3/60 defaults when config is missing", () => {
47
+ const out = rewriteFlappingTriggers(
48
+ def([{ event: LEGACY_FLAPPING_EVENT }]),
49
+ );
50
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
51
+ expect(triggers[0]!["window"]).toEqual({
52
+ count: DEFAULT_FLAPPING_COUNT,
53
+ minutes: DEFAULT_FLAPPING_MINUTES,
54
+ refire: "once",
55
+ });
56
+ });
57
+
58
+ it("applies defaults for a PARTIAL config", () => {
59
+ const out = rewriteFlappingTriggers(
60
+ def([
61
+ { event: LEGACY_FLAPPING_EVENT, config: { transitions: 7 } },
62
+ ]),
63
+ );
64
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
65
+ expect(triggers[0]!["window"]).toEqual({
66
+ count: 7,
67
+ minutes: DEFAULT_FLAPPING_MINUTES,
68
+ refire: "once",
69
+ });
70
+ });
71
+
72
+ it("replaces a pre-existing filter with the canonical one (safe option) and reports it", () => {
73
+ const out = rewriteFlappingTriggers(
74
+ def([
75
+ {
76
+ event: LEGACY_FLAPPING_EVENT,
77
+ filter: 'trigger.payload.systemId == "sys-1"',
78
+ config: { transitions: 3, windowMinutes: 60 },
79
+ },
80
+ ]),
81
+ );
82
+ expect(out.replacedFilter).toBe(true);
83
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
84
+ expect(triggers[0]!["filter"]).toBe(FLAPPING_FILTER);
85
+ });
86
+
87
+ it("preserves non-flapping triggers untouched", () => {
88
+ const out = rewriteFlappingTriggers(
89
+ def([
90
+ { event: "healthcheck.check_failed" },
91
+ { event: LEGACY_FLAPPING_EVENT, config: { transitions: 2 } },
92
+ ]),
93
+ );
94
+ expect(out.rewritten).toBe(1);
95
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
96
+ expect(triggers[0]).toEqual({ event: "healthcheck.check_failed" });
97
+ expect(triggers[1]!["event"]).toBe(HEALTH_CHANGED_EVENT);
98
+ });
99
+
100
+ it("is idempotent: a definition with no flapping trigger is left untouched", () => {
101
+ const input = def([{ event: HEALTH_CHANGED_EVENT, window: { count: 3, minutes: 60, refire: "once" } }]);
102
+ const out = rewriteFlappingTriggers(input);
103
+ expect(out.rewritten).toBe(0);
104
+ expect(out.definition).toBe(input); // same reference, untouched
105
+ });
106
+
107
+ it("rewrites multiple flapping triggers in one definition", () => {
108
+ const out = rewriteFlappingTriggers(
109
+ def([
110
+ { id: "a", event: LEGACY_FLAPPING_EVENT, config: { transitions: 2, windowMinutes: 10 } },
111
+ { id: "b", event: LEGACY_FLAPPING_EVENT },
112
+ ]),
113
+ );
114
+ expect(out.rewritten).toBe(2);
115
+ const triggers = out.definition["triggers"] as Record<string, unknown>[];
116
+ expect(triggers[0]!["window"]).toEqual({ count: 2, minutes: 10, refire: "once" });
117
+ expect(triggers[1]!["window"]).toEqual({
118
+ count: DEFAULT_FLAPPING_COUNT,
119
+ minutes: DEFAULT_FLAPPING_MINUTES,
120
+ refire: "once",
121
+ });
122
+ });
123
+ });