@checkstack/automation-backend 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +544 -0
- package/drizzle/0003_sparkling_xorn.sql +17 -0
- package/drizzle/0004_cultured_spyke.sql +2 -0
- package/drizzle/0005_classy_the_hand.sql +19 -0
- package/drizzle/0006_burly_wallop.sql +10 -0
- package/drizzle/0007_nappy_jackal.sql +1 -0
- package/drizzle/0008_remove_seeded_auto_incident_automations.sql +13 -0
- package/drizzle/0009_steady_liz_osborn.sql +12 -0
- package/drizzle/0010_chunky_changeling.sql +2 -0
- package/drizzle/meta/0003_snapshot.json +1007 -0
- package/drizzle/meta/0004_snapshot.json +1028 -0
- package/drizzle/meta/0005_snapshot.json +1164 -0
- package/drizzle/meta/0006_snapshot.json +1261 -0
- package/drizzle/meta/0007_snapshot.json +1215 -0
- package/drizzle/meta/0008_snapshot.json +1215 -0
- package/drizzle/meta/0009_snapshot.json +1328 -0
- package/drizzle/meta/0010_snapshot.json +1349 -0
- package/drizzle/meta/_journal.json +56 -0
- package/package.json +23 -12
- package/src/action-types.ts +23 -0
- package/src/artifact-store.ts +16 -1
- package/src/automation-store.test.ts +143 -0
- package/src/automation-store.ts +30 -8
- package/src/builtin-triggers.test.ts +77 -74
- package/src/builtin-triggers.ts +105 -108
- package/src/dispatch/action-kind.ts +2 -0
- package/src/dispatch/assemble-get-service.ts +31 -0
- package/src/dispatch/cancel-resurrect.test.ts +147 -0
- package/src/dispatch/concurrency-race.test.ts +255 -0
- package/src/dispatch/concurrency-scope.test.ts +166 -0
- package/src/dispatch/condition.ts +24 -5
- package/src/dispatch/dwell-queue.ts +65 -0
- package/src/dispatch/dwell-store.ts +154 -0
- package/src/dispatch/dwell.it.test.ts +142 -0
- package/src/dispatch/dwell.test.ts +799 -0
- package/src/dispatch/dwell.ts +257 -0
- package/src/dispatch/engine.test.ts +189 -2
- package/src/dispatch/engine.ts +555 -9
- package/src/dispatch/entity-scope.test.ts +176 -0
- package/src/dispatch/get-service-wiring.test.ts +318 -0
- package/src/dispatch/numeric.test.ts +71 -0
- package/src/dispatch/numeric.ts +96 -0
- package/src/dispatch/render.test.ts +34 -0
- package/src/dispatch/render.ts +31 -11
- package/src/dispatch/reseed-run-secrets.ts +230 -0
- package/src/dispatch/run-secret-registry.test.ts +189 -0
- package/src/dispatch/run-secret-registry.ts +247 -0
- package/src/dispatch/run-state-masking.test.ts +376 -0
- package/src/dispatch/run-state-store.ts +95 -38
- package/src/dispatch/run-state.ts +226 -59
- package/src/dispatch/scope-artifact-masking.test.ts +138 -0
- package/src/dispatch/secret-ref-ids.test.ts +19 -0
- package/src/dispatch/secret-ref-ids.ts +17 -0
- package/src/dispatch/snapshots.test.ts +86 -0
- package/src/dispatch/snapshots.ts +79 -0
- package/src/dispatch/stage1-router.test.ts +324 -0
- package/src/dispatch/stage1-router.ts +152 -0
- package/src/dispatch/stage1.it.test.ts +84 -0
- package/src/dispatch/stage2-dispatch.test.ts +285 -0
- package/src/dispatch/stage2-dispatch.ts +207 -0
- package/src/dispatch/stage2-stalled.it.test.ts +132 -0
- package/src/dispatch/stalled-sweeper.test.ts +197 -0
- package/src/dispatch/stalled-sweeper.ts +112 -5
- package/src/dispatch/state-scope.test.ts +234 -0
- package/src/dispatch/state-scope.ts +322 -0
- package/src/dispatch/structured-conditions.test.ts +246 -0
- package/src/dispatch/structured-conditions.ts +146 -0
- package/src/dispatch/test-fixtures.ts +306 -38
- package/src/dispatch/trigger-fanin.test.ts +111 -0
- package/src/dispatch/trigger-subscriber.ts +316 -14
- package/src/dispatch/types.ts +263 -8
- package/src/dispatch/wait-timeout-queue.ts +89 -0
- package/src/dispatch/wait-until-entity-wake.test.ts +544 -0
- package/src/dispatch/wait-until.test.ts +540 -0
- package/src/dispatch/wake-refs.test.ts +158 -0
- package/src/dispatch/wake-refs.ts +348 -0
- package/src/dispatch/window-gate.test.ts +513 -0
- package/src/dispatch/window-store.test.ts +162 -0
- package/src/dispatch/window-store.ts +102 -0
- package/src/entity/change-derivers.test.ts +148 -0
- package/src/entity/change-derivers.ts +143 -0
- package/src/entity/change-emitter.test.ts +66 -0
- package/src/entity/change-emitter.ts +76 -0
- package/src/entity/create-handle.ts +344 -0
- package/src/entity/cross-pod-read-consistency.it.test.ts +281 -0
- package/src/entity/define-entity.ts +157 -0
- package/src/entity/diff.test.ts +57 -0
- package/src/entity/diff.ts +54 -0
- package/src/entity/entity-store.test.ts +30 -0
- package/src/entity/entity-store.ts +171 -0
- package/src/entity/extension-point.ts +56 -0
- package/src/entity/fake-entity-store.ts +130 -0
- package/src/entity/hook.ts +19 -0
- package/src/entity/index.ts +50 -0
- package/src/entity/mutate-handle.test.ts +517 -0
- package/src/entity/on-entity-changed.test.ts +189 -0
- package/src/entity/on-entity-changed.ts +214 -0
- package/src/entity/registry.test.ts +181 -0
- package/src/entity/registry.ts +200 -0
- package/src/entity/stable-stringify.test.ts +55 -0
- package/src/entity/stable-stringify.ts +49 -0
- package/src/entity/wake-index.it.test.ts +251 -0
- package/src/entity/with-entity-write.test.ts +100 -0
- package/src/entity/with-entity-write.ts +69 -0
- package/src/entity-driven-trigger.ts +46 -0
- package/src/extension-points.ts +35 -0
- package/src/gitops-docs.test.ts +215 -0
- package/src/gitops-docs.ts +151 -0
- package/src/gitops-kinds.test.ts +174 -0
- package/src/gitops-kinds.ts +137 -0
- package/src/index.ts +355 -11
- package/src/migration/flapping-to-window.test.ts +123 -0
- package/src/migration/flapping-to-window.ts +205 -0
- package/src/router.test.ts +182 -1
- package/src/router.ts +73 -2
- package/src/schema.ts +236 -3
- package/src/script-test-replay.test.ts +88 -0
- package/src/script-test-replay.ts +100 -0
- package/src/script-test-shell-env.test.ts +41 -0
- package/src/script-test-shell-env.ts +89 -0
- package/src/script-test.test.ts +386 -0
- package/src/script-test.ts +258 -0
- package/src/trigger-registry.ts +2 -0
- package/src/validate-definition.test.ts +1 -0
- package/tsconfig.json +24 -0
package/src/index.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createBackendPlugin,
|
|
3
3
|
coreServices,
|
|
4
|
+
withXactLock,
|
|
5
|
+
type SafeDatabase,
|
|
4
6
|
} from "@checkstack/backend-api";
|
|
5
7
|
import {
|
|
6
8
|
automationAccess,
|
|
@@ -12,7 +14,19 @@ import {
|
|
|
12
14
|
import { resolveRoute, extractErrorMessage } from "@checkstack/common";
|
|
13
15
|
import type { PluginMetadata } from "@checkstack/common";
|
|
14
16
|
import { registerSearchProvider } from "@checkstack/command-backend";
|
|
15
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
createDefaultFilterRegistry,
|
|
19
|
+
type FilterRegistry,
|
|
20
|
+
} from "@checkstack/template-engine";
|
|
21
|
+
import { HealthCheckApi } from "@checkstack/healthcheck-common";
|
|
22
|
+
import { entityKindExtensionPoint } from "@checkstack/gitops-backend";
|
|
23
|
+
import { CHECKSTACK_API_VERSION } from "@checkstack/gitops-common";
|
|
24
|
+
import {
|
|
25
|
+
reconcileAutomation,
|
|
26
|
+
deleteAutomationEntity,
|
|
27
|
+
} from "./gitops-kinds";
|
|
28
|
+
import { registerAutomationGitOpsDocumentation } from "./gitops-docs";
|
|
29
|
+
import { AutomationDefinitionSchema } from "@checkstack/automation-common";
|
|
16
30
|
|
|
17
31
|
import type {
|
|
18
32
|
ActionDefinition,
|
|
@@ -29,12 +43,36 @@ import { createArtifactStore } from "./artifact-store";
|
|
|
29
43
|
import { createAutomationStore } from "./automation-store";
|
|
30
44
|
import { createAutomationRouter } from "./router";
|
|
31
45
|
import { runWebhookSubscriptionMigration } from "./migration/from-webhook-subscriptions";
|
|
46
|
+
import { runFlappingAutomationMigration } from "./migration/flapping-to-window";
|
|
32
47
|
import {
|
|
33
48
|
startDelayQueueConsumer,
|
|
34
49
|
type DelayQueueConsumer,
|
|
35
50
|
} from "./dispatch/delay-queue";
|
|
51
|
+
import {
|
|
52
|
+
startDwellQueueConsumer,
|
|
53
|
+
type DwellQueueConsumer,
|
|
54
|
+
} from "./dispatch/dwell-queue";
|
|
55
|
+
import {
|
|
56
|
+
startWaitTimeoutQueueConsumer,
|
|
57
|
+
type WaitTimeoutQueueConsumer,
|
|
58
|
+
} from "./dispatch/wait-timeout-queue";
|
|
59
|
+
import {
|
|
60
|
+
startDispatchQueueConsumer,
|
|
61
|
+
type DispatchQueueConsumer,
|
|
62
|
+
} from "./dispatch/stage2-dispatch";
|
|
63
|
+
import {
|
|
64
|
+
startStage1Router,
|
|
65
|
+
type Stage1Router,
|
|
66
|
+
} from "./dispatch/stage1-router";
|
|
67
|
+
import { createDwellStore } from "./dispatch/dwell-store";
|
|
68
|
+
import { createWindowStore } from "./dispatch/window-store";
|
|
36
69
|
import { createRunStore } from "./dispatch/run-state";
|
|
37
70
|
import { createRunStateStore } from "./dispatch/run-state-store";
|
|
71
|
+
import { createRunSecretRegistry } from "./dispatch/run-secret-registry";
|
|
72
|
+
import {
|
|
73
|
+
SECRET_RESOLVER_REF_ID,
|
|
74
|
+
CONNECTION_STORE_REF_ID,
|
|
75
|
+
} from "./dispatch/secret-ref-ids";
|
|
38
76
|
import {
|
|
39
77
|
startStalledSweeper,
|
|
40
78
|
type StalledSweeper,
|
|
@@ -44,10 +82,12 @@ import {
|
|
|
44
82
|
type TriggerSubscriptions,
|
|
45
83
|
} from "./dispatch/trigger-subscriber";
|
|
46
84
|
import type { DispatchDeps } from "./dispatch/types";
|
|
85
|
+
import { assembleDispatchGetService } from "./dispatch/assemble-get-service";
|
|
47
86
|
import {
|
|
48
87
|
automationActionExtensionPoint,
|
|
49
88
|
automationArtifactStoreRef,
|
|
50
89
|
automationArtifactTypeExtensionPoint,
|
|
90
|
+
automationFilterExtensionPoint,
|
|
51
91
|
automationRegistriesRef,
|
|
52
92
|
automationTriggerExtensionPoint,
|
|
53
93
|
} from "./extension-points";
|
|
@@ -55,6 +95,19 @@ import {
|
|
|
55
95
|
registerBuiltinTriggerConsumer,
|
|
56
96
|
registerBuiltinTriggers,
|
|
57
97
|
} from "./builtin-triggers";
|
|
98
|
+
import {
|
|
99
|
+
createChangeDeriverRegistry,
|
|
100
|
+
createChangeEmitter,
|
|
101
|
+
createEntityChangedSubscriptions,
|
|
102
|
+
createEntityRegistry,
|
|
103
|
+
createEntityStore,
|
|
104
|
+
entityExtensionPoint,
|
|
105
|
+
type ChangeDeriverRegistry,
|
|
106
|
+
type ChangeEmitter,
|
|
107
|
+
type EntityChangedSubscriptions,
|
|
108
|
+
type EntityRegistry,
|
|
109
|
+
} from "./entity";
|
|
110
|
+
import { ENTITY_CHANGED_HOOK } from "./entity/hook";
|
|
58
111
|
import {
|
|
59
112
|
createNotifyUserAction,
|
|
60
113
|
logAction,
|
|
@@ -73,21 +126,75 @@ interface EnvStash {
|
|
|
73
126
|
artifactTypeRegistry: ArtifactTypeRegistry;
|
|
74
127
|
dispatchDeps: DispatchDeps;
|
|
75
128
|
automationStore: ReturnType<typeof createAutomationStore>;
|
|
129
|
+
entityRegistry: EntityRegistry;
|
|
130
|
+
entityChangeEmitter: ChangeEmitter;
|
|
131
|
+
entityChangedSubscriptions: EntityChangedSubscriptions;
|
|
132
|
+
changeDerivers: ChangeDeriverRegistry;
|
|
76
133
|
triggerSubscriptions?: TriggerSubscriptions;
|
|
77
134
|
stalledSweeper?: StalledSweeper;
|
|
78
135
|
delayConsumer?: DelayQueueConsumer;
|
|
136
|
+
dwellConsumer?: DwellQueueConsumer;
|
|
137
|
+
waitTimeoutConsumer?: WaitTimeoutQueueConsumer;
|
|
138
|
+
dispatchConsumer?: DispatchQueueConsumer;
|
|
139
|
+
stage1Router?: Stage1Router;
|
|
79
140
|
}
|
|
80
141
|
|
|
81
142
|
export default createBackendPlugin({
|
|
82
143
|
metadata: pluginMetadata,
|
|
83
144
|
|
|
84
145
|
register(env) {
|
|
146
|
+
// Mutable DB ref — populated in init(), consumed by the GitOps
|
|
147
|
+
// reconcile/delete closures (only called during sync, after init).
|
|
148
|
+
let gitopsDb: SafeDatabase<typeof schema> | undefined;
|
|
149
|
+
|
|
85
150
|
const triggerRegistry = createTriggerRegistry();
|
|
86
151
|
const actionRegistry = createActionRegistry();
|
|
87
152
|
const artifactTypeRegistry = createArtifactTypeRegistry();
|
|
153
|
+
// Shared filter registry — seeded with the built-in defaults (incl.
|
|
154
|
+
// the Wave-2 duration helpers) and extended by plugins via the
|
|
155
|
+
// filter extension point in Phase 1. The dispatch engine reads from
|
|
156
|
+
// this same instance, so plugin filters are live by `init()`.
|
|
157
|
+
const filterRegistry: FilterRegistry = createDefaultFilterRegistry();
|
|
158
|
+
|
|
159
|
+
// Run-scoped secret registry — created here in `register()` (it has no
|
|
160
|
+
// deps) so the SAME instance backs both the dispatch masking choke
|
|
161
|
+
// points (wired in `init()`) and the entity-store masking choke point
|
|
162
|
+
// (run-originated entity writes mask through this in the handle).
|
|
163
|
+
const secretRegistry = createRunSecretRegistry();
|
|
164
|
+
|
|
165
|
+
// Entity state machine (reactive automation engine §4). The change
|
|
166
|
+
// emitter buffers until `emitHook` is wired in `afterPluginsReady`
|
|
167
|
+
// (§3.7); the registry exposes `defineEntity` / `declareNonReactiveState`
|
|
168
|
+
// through the extension point below, callable from other plugins'
|
|
169
|
+
// `register`/`init`. The DB-backed store is bound in `init()` (after
|
|
170
|
+
// migrations run).
|
|
171
|
+
const entityChangeEmitter = createChangeEmitter();
|
|
172
|
+
const entityRegistry = createEntityRegistry({
|
|
173
|
+
secretRegistry,
|
|
174
|
+
emitter: entityChangeEmitter,
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
// Reactive dispatch pipeline (reactive automation engine §7). The
|
|
178
|
+
// change-deriver registry maps a kind's change → trigger event id(s)
|
|
179
|
+
// for Stage-1 routing; per-domain derivers are registered in Phase 4.
|
|
180
|
+
// The entity-changed subscription service rides ENTITY_CHANGED (filtered
|
|
181
|
+
// by kind) so other plugins can react without touching the internal hook
|
|
182
|
+
// (§6.1). Both buffer registrations made before afterPluginsReady wires
|
|
183
|
+
// the real `onHook`.
|
|
184
|
+
const changeDerivers = createChangeDeriverRegistry();
|
|
185
|
+
const entityChangedSubscriptions = createEntityChangedSubscriptions();
|
|
88
186
|
|
|
89
187
|
env.registerAccessRules(automationAccessRules);
|
|
90
188
|
|
|
189
|
+
// Phase 1: register the entity extension point so other plugins can
|
|
190
|
+
// resolve it and call `defineEntity` during their own register/init.
|
|
191
|
+
env.registerExtensionPoint(entityExtensionPoint, {
|
|
192
|
+
defineEntity: entityRegistry.defineEntity,
|
|
193
|
+
declareNonReactiveState: entityRegistry.declareNonReactiveState,
|
|
194
|
+
onEntityChanged: entityChangedSubscriptions.onEntityChanged,
|
|
195
|
+
registerChangeDeriver: (input) => changeDerivers.register(input),
|
|
196
|
+
});
|
|
197
|
+
|
|
91
198
|
env.registerExtensionPoint(automationTriggerExtensionPoint, {
|
|
92
199
|
registerTrigger: <TPayload, TConfig = void>(
|
|
93
200
|
definition: TriggerDefinition<TPayload, TConfig>,
|
|
@@ -124,6 +231,64 @@ export default createBackendPlugin({
|
|
|
124
231
|
},
|
|
125
232
|
});
|
|
126
233
|
|
|
234
|
+
// Filters registered by plugins in Phase 1 are collected here and
|
|
235
|
+
// applied (with collision-warning) in `init()` where a logger is
|
|
236
|
+
// available. Collecting first keeps `register()` logger-free.
|
|
237
|
+
const pendingFilters: Array<{
|
|
238
|
+
name: string;
|
|
239
|
+
filter: Parameters<FilterRegistry["register"]>[1];
|
|
240
|
+
pluginId: string;
|
|
241
|
+
}> = [];
|
|
242
|
+
env.registerExtensionPoint(automationFilterExtensionPoint, {
|
|
243
|
+
registerFilter: (definition, metadata) => {
|
|
244
|
+
pendingFilters.push({
|
|
245
|
+
name: definition.name,
|
|
246
|
+
filter: definition.filter,
|
|
247
|
+
pluginId: metadata.pluginId,
|
|
248
|
+
});
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
// GitOps `Automation` kind. The DB isn't available until init(), so a
|
|
253
|
+
// mutable ref is populated there and read by the reconcile closures
|
|
254
|
+
// (only invoked during sync, well after init) — mirrors catalog-backend.
|
|
255
|
+
const kindRegistry = env.getExtensionPoint(entityKindExtensionPoint);
|
|
256
|
+
kindRegistry.registerKind({
|
|
257
|
+
apiVersion: CHECKSTACK_API_VERSION,
|
|
258
|
+
kind: "Automation",
|
|
259
|
+
specSchema: AutomationDefinitionSchema,
|
|
260
|
+
reconcile: async ({ entity, existingEntityId, context }) => {
|
|
261
|
+
if (!gitopsDb) throw new Error("Automation database not initialized");
|
|
262
|
+
return reconcileAutomation(gitopsDb, {
|
|
263
|
+
entity,
|
|
264
|
+
existingEntityId,
|
|
265
|
+
logger: context.logger,
|
|
266
|
+
});
|
|
267
|
+
},
|
|
268
|
+
delete: async ({ entityId, context }) => {
|
|
269
|
+
if (!gitopsDb) throw new Error("Automation database not initialized");
|
|
270
|
+
await deleteAutomationEntity(gitopsDb, {
|
|
271
|
+
entityId,
|
|
272
|
+
logger: context.logger,
|
|
273
|
+
});
|
|
274
|
+
},
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// Register the GitOps spec-schema documentation PROVIDER for the
|
|
278
|
+
// `Automation` kind HERE in register() (not afterPluginsReady). It is a
|
|
279
|
+
// LAZY provider: it re-reads the trigger/action registries on every
|
|
280
|
+
// kind-browser query (`describeKinds()`), so it needs no populated
|
|
281
|
+
// registries at registration time — registering it early, before any
|
|
282
|
+
// init / afterPluginsReady ordering, guarantees it is always present.
|
|
283
|
+
// Surfaces each trigger's / provider action's config schema in the Kind
|
|
284
|
+
// Registry, conditioned on the chosen `triggers[].event` /
|
|
285
|
+
// `actions[].action`.
|
|
286
|
+
registerAutomationGitOpsDocumentation({
|
|
287
|
+
kindRegistry,
|
|
288
|
+
triggerRegistry,
|
|
289
|
+
actionRegistry,
|
|
290
|
+
});
|
|
291
|
+
|
|
127
292
|
env.registerInit({
|
|
128
293
|
schema,
|
|
129
294
|
deps: {
|
|
@@ -132,6 +297,7 @@ export default createBackendPlugin({
|
|
|
132
297
|
rpcClient: coreServices.rpcClient,
|
|
133
298
|
queueManager: coreServices.queueManager,
|
|
134
299
|
signalService: coreServices.signalService,
|
|
300
|
+
advisoryLock: coreServices.advisoryLock,
|
|
135
301
|
},
|
|
136
302
|
init: async ({
|
|
137
303
|
logger,
|
|
@@ -140,14 +306,37 @@ export default createBackendPlugin({
|
|
|
140
306
|
rpcClient,
|
|
141
307
|
queueManager,
|
|
142
308
|
signalService,
|
|
309
|
+
advisoryLock,
|
|
143
310
|
}) => {
|
|
144
311
|
logger.debug("⚙️ Initializing Automation Backend...");
|
|
145
312
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
313
|
+
// Populate the mutable DB ref the GitOps reconcile closures read.
|
|
314
|
+
gitopsDb = database as SafeDatabase<typeof schema>;
|
|
315
|
+
|
|
316
|
+
// Run-scoped secret registry: created in `register()` (the SAME
|
|
317
|
+
// instance) so it accumulates every secret value resolved during a
|
|
318
|
+
// run and every persistence choke point (run store step/run output,
|
|
319
|
+
// run-state scope snapshot, artifact data, AND run-originated entity
|
|
320
|
+
// writes) masks before write.
|
|
321
|
+
const artifactStore = createArtifactStore(database, secretRegistry);
|
|
322
|
+
const runStore = createRunStore(database, logger, secretRegistry);
|
|
323
|
+
const runStateStore = createRunStateStore(
|
|
324
|
+
database,
|
|
325
|
+
advisoryLock,
|
|
326
|
+
secretRegistry,
|
|
327
|
+
);
|
|
328
|
+
const dwellStore = createDwellStore(database);
|
|
329
|
+
const windowStore = createWindowStore(database);
|
|
149
330
|
const automationStore = createAutomationStore(database);
|
|
150
331
|
|
|
332
|
+
// Bind the DB-backed transition store to the registry (the extension
|
|
333
|
+
// point impl registered in `register()` forwards through it). Model B:
|
|
334
|
+
// the transition store owns the tx + `entity_transitions` log for
|
|
335
|
+
// EVERY kind. Bound here in `init()` — after migrations have run — so
|
|
336
|
+
// the table exists.
|
|
337
|
+
const entityStore = createEntityStore(database);
|
|
338
|
+
entityRegistry.setStore({ store: entityStore });
|
|
339
|
+
|
|
151
340
|
env.registerService(automationArtifactStoreRef, artifactStore);
|
|
152
341
|
env.registerService(automationRegistriesRef, {
|
|
153
342
|
triggers: triggerRegistry,
|
|
@@ -188,9 +377,21 @@ export default createBackendPlugin({
|
|
|
188
377
|
);
|
|
189
378
|
await registerBuiltinTriggerConsumer({ queueManager, logger });
|
|
190
379
|
|
|
380
|
+
// Apply plugin-contributed filters collected in register(),
|
|
381
|
+
// skipping any that would shadow a built-in (warn, don't clobber).
|
|
382
|
+
for (const pf of pendingFilters) {
|
|
383
|
+
if (filterRegistry.has(pf.name)) {
|
|
384
|
+
logger.warn(
|
|
385
|
+
`Plugin ${pf.pluginId} tried to register filter "${pf.name}" which already exists; skipping.`,
|
|
386
|
+
);
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
filterRegistry.register(pf.name, pf.filter);
|
|
390
|
+
}
|
|
391
|
+
|
|
191
392
|
const dispatchDeps: DispatchDeps = {
|
|
192
393
|
logger,
|
|
193
|
-
filters:
|
|
394
|
+
filters: filterRegistry,
|
|
194
395
|
registries: {
|
|
195
396
|
triggers: triggerRegistry,
|
|
196
397
|
actions: actionRegistry,
|
|
@@ -199,12 +400,40 @@ export default createBackendPlugin({
|
|
|
199
400
|
artifactStore,
|
|
200
401
|
runStore,
|
|
201
402
|
runStateStore,
|
|
403
|
+
dwellStore,
|
|
404
|
+
windowStore,
|
|
202
405
|
queueManager,
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
406
|
+
// Sensing-layer scope pre-resolution reads live health state
|
|
407
|
+
// through this client. forPlugin is lazy; the actual RPC only
|
|
408
|
+
// fires at evaluation time.
|
|
409
|
+
healthCheckClient: rpcClient.forPlugin(HealthCheckApi),
|
|
410
|
+
// Kind-agnostic entity resolver for reactive `wait_until` wake
|
|
411
|
+
// re-evaluation (Model B): the registry routes each kind to its
|
|
412
|
+
// plugin `read` accessor. Unknown kinds
|
|
413
|
+
// yield `undefined` (enrichment leaves them unresolved, fail-open).
|
|
414
|
+
// This is what lets a wait on `state.<kind>.<id>` (incident, slo, …)
|
|
415
|
+
// re-evaluate correctly when that kind changes (not just health).
|
|
416
|
+
entityResolverFor: (kind) => entityRegistry.entityResolverFor(kind),
|
|
417
|
+
// Registry-backed resolution of provider-action deps (connection
|
|
418
|
+
// store, secret resolver, ...) at execute time. Safe here because
|
|
419
|
+
// dispatch only runs from afterPluginsReady onward, by which point
|
|
420
|
+
// every service is registered. `env.getService` resolves through
|
|
421
|
+
// the real ServiceRegistry and throws clearly on a missing ref.
|
|
422
|
+
getService: assembleDispatchGetService({
|
|
423
|
+
envGetService: env.getService,
|
|
424
|
+
}),
|
|
425
|
+
// Run-wide secret masking: the engine wraps each run's getService
|
|
426
|
+
// to register resolved secrets here, and the run store masks step
|
|
427
|
+
// / run output before persistence.
|
|
428
|
+
secretRegistry,
|
|
429
|
+
secretResolverRefId: SECRET_RESOLVER_REF_ID,
|
|
430
|
+
connectionStoreRefId: CONNECTION_STORE_REF_ID,
|
|
431
|
+
// Serialize the concurrency-mode check-then-create with a
|
|
432
|
+
// transaction-scoped advisory lock (blocks until granted,
|
|
433
|
+
// auto-releases at COMMIT) so racing fires can't double-run a
|
|
434
|
+
// single-mode automation.
|
|
435
|
+
withConcurrencyLock: <T>(key: string, fn: () => Promise<T>) =>
|
|
436
|
+
withXactLock({ db: database, key, fn: () => fn() }),
|
|
208
437
|
};
|
|
209
438
|
|
|
210
439
|
const stash = env as unknown as EnvStash;
|
|
@@ -213,6 +442,10 @@ export default createBackendPlugin({
|
|
|
213
442
|
stash.artifactTypeRegistry = artifactTypeRegistry;
|
|
214
443
|
stash.dispatchDeps = dispatchDeps;
|
|
215
444
|
stash.automationStore = automationStore;
|
|
445
|
+
stash.entityRegistry = entityRegistry;
|
|
446
|
+
stash.entityChangeEmitter = entityChangeEmitter;
|
|
447
|
+
stash.entityChangedSubscriptions = entityChangedSubscriptions;
|
|
448
|
+
stash.changeDerivers = changeDerivers;
|
|
216
449
|
|
|
217
450
|
const router = createAutomationRouter({
|
|
218
451
|
db: database,
|
|
@@ -260,19 +493,43 @@ export default createBackendPlugin({
|
|
|
260
493
|
env.registerCleanup(async () => {
|
|
261
494
|
const s = env as unknown as EnvStash;
|
|
262
495
|
await s.triggerSubscriptions?.dispose();
|
|
496
|
+
await s.stage1Router?.dispose();
|
|
497
|
+
await s.entityChangedSubscriptions?.disposeAll();
|
|
263
498
|
s.stalledSweeper?.stop();
|
|
264
499
|
await s.delayConsumer?.stop();
|
|
500
|
+
await s.dwellConsumer?.stop();
|
|
501
|
+
await s.waitTimeoutConsumer?.stop();
|
|
502
|
+
await s.dispatchConsumer?.stop();
|
|
265
503
|
});
|
|
266
504
|
|
|
267
505
|
logger.debug("✅ Automation Backend initialized.");
|
|
268
506
|
},
|
|
269
507
|
|
|
270
|
-
afterPluginsReady: async ({
|
|
508
|
+
afterPluginsReady: async ({
|
|
509
|
+
database,
|
|
510
|
+
logger,
|
|
511
|
+
onHook,
|
|
512
|
+
emitHook,
|
|
513
|
+
rpcClient,
|
|
514
|
+
}) => {
|
|
271
515
|
const stash = env as unknown as EnvStash;
|
|
272
516
|
const triggers = stash.triggerRegistry.getTriggers();
|
|
273
517
|
const actions = stash.actionRegistry.getActions();
|
|
274
518
|
const artifactTypes = stash.artifactTypeRegistry.getArtifactTypes();
|
|
275
519
|
|
|
520
|
+
// Wire the deferred entity-change emitter to the real `emitHook`
|
|
521
|
+
// (only injectable here — §3.7). Any change events buffered during
|
|
522
|
+
// the init / afterPluginsReady window are flushed in order now, so
|
|
523
|
+
// there is no silent no-emit gap.
|
|
524
|
+
await stash.entityChangeEmitter.wire((payload) =>
|
|
525
|
+
emitHook(ENTITY_CHANGED_HOOK, payload),
|
|
526
|
+
);
|
|
527
|
+
|
|
528
|
+
// Wire the public cross-plugin entity-change subscription service
|
|
529
|
+
// (§6.1). Subscriptions registered by other plugins during their
|
|
530
|
+
// register/init are bound to the real `onHook` now.
|
|
531
|
+
stash.entityChangedSubscriptions.wire({ onHook, logger });
|
|
532
|
+
|
|
276
533
|
logger.debug(
|
|
277
534
|
`⚙️ Registered ${triggers.length} automation triggers${
|
|
278
535
|
triggers.length > 0
|
|
@@ -313,6 +570,46 @@ export default createBackendPlugin({
|
|
|
313
570
|
logger,
|
|
314
571
|
});
|
|
315
572
|
|
|
573
|
+
// `for:` dwell: register the consumer that fires when a dwell's
|
|
574
|
+
// scheduled job pops, re-confirming state before starting the run.
|
|
575
|
+
stash.dwellConsumer = await startDwellQueueConsumer({
|
|
576
|
+
deps: stash.dispatchDeps,
|
|
577
|
+
automationStore: stash.automationStore,
|
|
578
|
+
logger,
|
|
579
|
+
});
|
|
580
|
+
|
|
581
|
+
// Reactive `wait_until` timeout timer: the consumer that fires when
|
|
582
|
+
// a suspended wait's single deadline job pops, applying the
|
|
583
|
+
// continue/fail-on-timeout policy. Reactive waits are otherwise woken
|
|
584
|
+
// by Stage-1 routing on a relevant ENTITY_CHANGED (no polling).
|
|
585
|
+
stash.waitTimeoutConsumer = await startWaitTimeoutQueueConsumer({
|
|
586
|
+
deps: stash.dispatchDeps,
|
|
587
|
+
automationStore: stash.automationStore,
|
|
588
|
+
logger,
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
// Stage-2 dispatch fan-out: the consumer that runs each per-run
|
|
592
|
+
// dispatch job enqueued by Stage-1 routing (reason: trigger →
|
|
593
|
+
// dispatchTrigger; reason: wake → resume the suspended wait_until).
|
|
594
|
+
stash.dispatchConsumer = await startDispatchQueueConsumer({
|
|
595
|
+
deps: stash.dispatchDeps,
|
|
596
|
+
automationStore: stash.automationStore,
|
|
597
|
+
changeDerivers: stash.changeDerivers,
|
|
598
|
+
logger,
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
// Stage-1 routing: claim each ENTITY_CHANGED on the
|
|
602
|
+
// `automation-entity-route` work-queue (exactly one instance), do
|
|
603
|
+
// cheap indexed routing (wake-index intersection + trigger-event
|
|
604
|
+
// derivation), and enqueue Stage-2 jobs.
|
|
605
|
+
stash.stage1Router = await startStage1Router({
|
|
606
|
+
deps: stash.dispatchDeps,
|
|
607
|
+
automationStore: stash.automationStore,
|
|
608
|
+
changeDerivers: stash.changeDerivers,
|
|
609
|
+
onHook,
|
|
610
|
+
logger,
|
|
611
|
+
});
|
|
612
|
+
|
|
316
613
|
// Restart safety + horizontal scaling: periodically scan for
|
|
317
614
|
// runs whose heartbeat is older than the threshold and resume
|
|
318
615
|
// them under an advisory lock.
|
|
@@ -340,6 +637,19 @@ export default createBackendPlugin({
|
|
|
340
637
|
);
|
|
341
638
|
}
|
|
342
639
|
|
|
640
|
+
// One-time migration: rewrite legacy `healthcheck.flapping_detected`
|
|
641
|
+
// triggers onto the generic windowed-count gate over
|
|
642
|
+
// `healthcheck.system_health_changed` (the flapping trigger + hook
|
|
643
|
+
// were removed). Idempotent — already-migrated / non-flapping rows are
|
|
644
|
+
// skipped — so it is safe to run on every boot.
|
|
645
|
+
try {
|
|
646
|
+
await runFlappingAutomationMigration({ db: database, logger });
|
|
647
|
+
} catch (error) {
|
|
648
|
+
logger.error(
|
|
649
|
+
`Flapping automation migration failed unexpectedly: ${extractErrorMessage(error, "unknown error")}`,
|
|
650
|
+
);
|
|
651
|
+
}
|
|
652
|
+
|
|
343
653
|
logger.debug("✅ Automation Backend afterPluginsReady complete.");
|
|
344
654
|
},
|
|
345
655
|
});
|
|
@@ -356,6 +666,37 @@ export {
|
|
|
356
666
|
automationArtifactStoreRef,
|
|
357
667
|
} from "./extension-points";
|
|
358
668
|
|
|
669
|
+
// Entity state machine — the typed path to reactive state. The internal
|
|
670
|
+
// ENTITY_CHANGED hook is intentionally NOT re-exported (§6.1).
|
|
671
|
+
export { entityExtensionPoint } from "./entity";
|
|
672
|
+
export { withEntityWrite, withEntityRemove } from "./entity";
|
|
673
|
+
export type {
|
|
674
|
+
EntityExtensionPoint,
|
|
675
|
+
DefineEntity,
|
|
676
|
+
DefineEntityInput,
|
|
677
|
+
DeclareNonReactiveState,
|
|
678
|
+
DeclareNonReactiveStateInput,
|
|
679
|
+
EntityHandle,
|
|
680
|
+
EntityMutationOpts,
|
|
681
|
+
EntityRead,
|
|
682
|
+
MutateInput,
|
|
683
|
+
RemoveInput,
|
|
684
|
+
EntityTx,
|
|
685
|
+
EntityChangeDeriver,
|
|
686
|
+
EntityChangePayloadMapper,
|
|
687
|
+
RegisterChangeDeriver,
|
|
688
|
+
OnEntityChanged,
|
|
689
|
+
OnEntityChangedInput,
|
|
690
|
+
EntityChangedHandler,
|
|
691
|
+
EntityChangedDelivery,
|
|
692
|
+
EntityChangedUnsubscribe,
|
|
693
|
+
} from "./entity";
|
|
694
|
+
|
|
695
|
+
// The validated entity-change payload (Phase 4 derivers + cross-plugin
|
|
696
|
+
// consumers type against this). Re-exported from automation-common so a
|
|
697
|
+
// domain plugin needs only the automation-backend dependency.
|
|
698
|
+
export type { EntityChanged } from "@checkstack/automation-common";
|
|
699
|
+
|
|
359
700
|
export type {
|
|
360
701
|
TriggerDefinition,
|
|
361
702
|
ActionDefinition,
|
|
@@ -371,9 +712,12 @@ export type {
|
|
|
371
712
|
TriggerTeardown,
|
|
372
713
|
} from "./action-types";
|
|
373
714
|
|
|
715
|
+
export { makeEntityDrivenTriggerSetup } from "./entity-driven-trigger";
|
|
716
|
+
|
|
374
717
|
export type { ArtifactStore, PersistedArtifact } from "./artifact-store";
|
|
375
718
|
export type { TriggerRegistry } from "./trigger-registry";
|
|
376
719
|
export type { ActionRegistry } from "./action-registry";
|
|
377
720
|
export type { ArtifactTypeRegistry } from "./artifact-type-registry";
|
|
378
721
|
export type { AutomationRegistries } from "./extension-points";
|
|
379
722
|
export type { AutomationStore } from "./automation-store";
|
|
723
|
+
export type { LoadedAutomation } from "./dispatch/types";
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_FLAPPING_COUNT,
|
|
5
|
+
DEFAULT_FLAPPING_MINUTES,
|
|
6
|
+
FLAPPING_FILTER,
|
|
7
|
+
HEALTH_CHANGED_EVENT,
|
|
8
|
+
LEGACY_FLAPPING_EVENT,
|
|
9
|
+
rewriteFlappingTriggers,
|
|
10
|
+
} from "./flapping-to-window";
|
|
11
|
+
|
|
12
|
+
function def(triggers: unknown[]): Record<string, unknown> {
|
|
13
|
+
return {
|
|
14
|
+
name: "Flapping",
|
|
15
|
+
triggers,
|
|
16
|
+
conditions: [],
|
|
17
|
+
actions: [{ action: "incident.create", config: {} }],
|
|
18
|
+
mode: "single",
|
|
19
|
+
max_runs: 1,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe("rewriteFlappingTriggers", () => {
|
|
24
|
+
it("maps {transitions, windowMinutes} → window {count, minutes, refire: once}", () => {
|
|
25
|
+
const out = rewriteFlappingTriggers(
|
|
26
|
+
def([
|
|
27
|
+
{
|
|
28
|
+
id: "flap",
|
|
29
|
+
event: LEGACY_FLAPPING_EVENT,
|
|
30
|
+
config: { transitions: 5, windowMinutes: 15 },
|
|
31
|
+
},
|
|
32
|
+
]),
|
|
33
|
+
);
|
|
34
|
+
expect(out.rewritten).toBe(1);
|
|
35
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
36
|
+
expect(triggers[0]).toEqual({
|
|
37
|
+
id: "flap",
|
|
38
|
+
event: HEALTH_CHANGED_EVENT,
|
|
39
|
+
filter: FLAPPING_FILTER,
|
|
40
|
+
window: { count: 5, minutes: 15, refire: "once" },
|
|
41
|
+
});
|
|
42
|
+
// config is dropped
|
|
43
|
+
expect("config" in triggers[0]!).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("applies 3/60 defaults when config is missing", () => {
|
|
47
|
+
const out = rewriteFlappingTriggers(
|
|
48
|
+
def([{ event: LEGACY_FLAPPING_EVENT }]),
|
|
49
|
+
);
|
|
50
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
51
|
+
expect(triggers[0]!["window"]).toEqual({
|
|
52
|
+
count: DEFAULT_FLAPPING_COUNT,
|
|
53
|
+
minutes: DEFAULT_FLAPPING_MINUTES,
|
|
54
|
+
refire: "once",
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("applies defaults for a PARTIAL config", () => {
|
|
59
|
+
const out = rewriteFlappingTriggers(
|
|
60
|
+
def([
|
|
61
|
+
{ event: LEGACY_FLAPPING_EVENT, config: { transitions: 7 } },
|
|
62
|
+
]),
|
|
63
|
+
);
|
|
64
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
65
|
+
expect(triggers[0]!["window"]).toEqual({
|
|
66
|
+
count: 7,
|
|
67
|
+
minutes: DEFAULT_FLAPPING_MINUTES,
|
|
68
|
+
refire: "once",
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("replaces a pre-existing filter with the canonical one (safe option) and reports it", () => {
|
|
73
|
+
const out = rewriteFlappingTriggers(
|
|
74
|
+
def([
|
|
75
|
+
{
|
|
76
|
+
event: LEGACY_FLAPPING_EVENT,
|
|
77
|
+
filter: 'trigger.payload.systemId == "sys-1"',
|
|
78
|
+
config: { transitions: 3, windowMinutes: 60 },
|
|
79
|
+
},
|
|
80
|
+
]),
|
|
81
|
+
);
|
|
82
|
+
expect(out.replacedFilter).toBe(true);
|
|
83
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
84
|
+
expect(triggers[0]!["filter"]).toBe(FLAPPING_FILTER);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it("preserves non-flapping triggers untouched", () => {
|
|
88
|
+
const out = rewriteFlappingTriggers(
|
|
89
|
+
def([
|
|
90
|
+
{ event: "healthcheck.check_failed" },
|
|
91
|
+
{ event: LEGACY_FLAPPING_EVENT, config: { transitions: 2 } },
|
|
92
|
+
]),
|
|
93
|
+
);
|
|
94
|
+
expect(out.rewritten).toBe(1);
|
|
95
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
96
|
+
expect(triggers[0]).toEqual({ event: "healthcheck.check_failed" });
|
|
97
|
+
expect(triggers[1]!["event"]).toBe(HEALTH_CHANGED_EVENT);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("is idempotent: a definition with no flapping trigger is left untouched", () => {
|
|
101
|
+
const input = def([{ event: HEALTH_CHANGED_EVENT, window: { count: 3, minutes: 60, refire: "once" } }]);
|
|
102
|
+
const out = rewriteFlappingTriggers(input);
|
|
103
|
+
expect(out.rewritten).toBe(0);
|
|
104
|
+
expect(out.definition).toBe(input); // same reference, untouched
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it("rewrites multiple flapping triggers in one definition", () => {
|
|
108
|
+
const out = rewriteFlappingTriggers(
|
|
109
|
+
def([
|
|
110
|
+
{ id: "a", event: LEGACY_FLAPPING_EVENT, config: { transitions: 2, windowMinutes: 10 } },
|
|
111
|
+
{ id: "b", event: LEGACY_FLAPPING_EVENT },
|
|
112
|
+
]),
|
|
113
|
+
);
|
|
114
|
+
expect(out.rewritten).toBe(2);
|
|
115
|
+
const triggers = out.definition["triggers"] as Record<string, unknown>[];
|
|
116
|
+
expect(triggers[0]!["window"]).toEqual({ count: 2, minutes: 10, refire: "once" });
|
|
117
|
+
expect(triggers[1]!["window"]).toEqual({
|
|
118
|
+
count: DEFAULT_FLAPPING_COUNT,
|
|
119
|
+
minutes: DEFAULT_FLAPPING_MINUTES,
|
|
120
|
+
refire: "once",
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
});
|