@blokjs/runner 0.2.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Blok.js +32 -3
- package/dist/Blok.js.map +1 -1
- package/dist/Configuration.d.ts +59 -5
- package/dist/Configuration.js +366 -96
- package/dist/Configuration.js.map +1 -1
- package/dist/ForEachNode.d.ts +59 -0
- package/dist/ForEachNode.js +522 -0
- package/dist/ForEachNode.js.map +1 -0
- package/dist/LoopMaxIterationsError.d.ts +11 -0
- package/dist/LoopMaxIterationsError.js +18 -0
- package/dist/LoopMaxIterationsError.js.map +1 -0
- package/dist/LoopNode.d.ts +36 -0
- package/dist/LoopNode.js +182 -0
- package/dist/LoopNode.js.map +1 -0
- package/dist/PayloadTooLargeError.d.ts +19 -0
- package/dist/PayloadTooLargeError.js +29 -0
- package/dist/PayloadTooLargeError.js.map +1 -0
- package/dist/RunCancelledError.d.ts +17 -0
- package/dist/RunCancelledError.js +25 -0
- package/dist/RunCancelledError.js.map +1 -0
- package/dist/Runner.d.ts +11 -1
- package/dist/Runner.js +9 -2
- package/dist/Runner.js.map +1 -1
- package/dist/RunnerSteps.js +648 -44
- package/dist/RunnerSteps.js.map +1 -1
- package/dist/RuntimeAdapterNode.d.ts +2 -1
- package/dist/RuntimeAdapterNode.js +2 -2
- package/dist/RuntimeAdapterNode.js.map +1 -1
- package/dist/RuntimeRegistry.d.ts +23 -2
- package/dist/RuntimeRegistry.js +31 -2
- package/dist/RuntimeRegistry.js.map +1 -1
- package/dist/SubworkflowNode.d.ts +181 -0
- package/dist/SubworkflowNode.js +479 -0
- package/dist/SubworkflowNode.js.map +1 -0
- package/dist/SwitchNode.d.ts +37 -0
- package/dist/SwitchNode.js +153 -0
- package/dist/SwitchNode.js.map +1 -0
- package/dist/TriggerBase.d.ts +178 -0
- package/dist/TriggerBase.js +1032 -5
- package/dist/TriggerBase.js.map +1 -1
- package/dist/TryCatchNode.d.ts +32 -0
- package/dist/TryCatchNode.js +207 -0
- package/dist/TryCatchNode.js.map +1 -0
- package/dist/WaitDispatchRequest.d.ts +38 -0
- package/dist/WaitDispatchRequest.js +13 -0
- package/dist/WaitDispatchRequest.js.map +1 -0
- package/dist/WaitNode.d.ts +23 -0
- package/dist/WaitNode.js +26 -0
- package/dist/WaitNode.js.map +1 -0
- package/dist/adapters/grpc/GrpcCodec.js +2 -2
- package/dist/adapters/grpc/GrpcRuntimeAdapter.d.ts +6 -4
- package/dist/adapters/grpc/GrpcRuntimeAdapter.js +6 -4
- package/dist/adapters/grpc/GrpcRuntimeAdapter.js.map +1 -1
- package/dist/adapters/grpc/types.d.ts +7 -5
- package/dist/adapters/grpc/types.js.map +1 -1
- package/dist/adapters/transport.d.ts +12 -41
- package/dist/adapters/transport.js +21 -70
- package/dist/adapters/transport.js.map +1 -1
- package/dist/cache/NodeResultCache.js +7 -0
- package/dist/cache/NodeResultCache.js.map +1 -1
- package/dist/concurrency/ConcurrencyBackend.d.ts +61 -0
- package/dist/concurrency/ConcurrencyBackend.js +20 -0
- package/dist/concurrency/ConcurrencyBackend.js.map +1 -0
- package/dist/concurrency/ConcurrencyLimitError.d.ts +37 -0
- package/dist/concurrency/ConcurrencyLimitError.js +16 -0
- package/dist/concurrency/ConcurrencyLimitError.js.map +1 -0
- package/dist/concurrency/NatsKvConcurrencyBackend.d.ts +64 -0
- package/dist/concurrency/NatsKvConcurrencyBackend.js +310 -0
- package/dist/concurrency/NatsKvConcurrencyBackend.js.map +1 -0
- package/dist/concurrency/QueueExpiredError.d.ts +40 -0
- package/dist/concurrency/QueueExpiredError.js +15 -0
- package/dist/concurrency/QueueExpiredError.js.map +1 -0
- package/dist/concurrency/RedisConcurrencyBackend.d.ts +64 -0
- package/dist/concurrency/RedisConcurrencyBackend.js +374 -0
- package/dist/concurrency/RedisConcurrencyBackend.js.map +1 -0
- package/dist/concurrency/createConcurrencyBackend.d.ts +24 -0
- package/dist/concurrency/createConcurrencyBackend.js +38 -0
- package/dist/concurrency/createConcurrencyBackend.js.map +1 -0
- package/dist/concurrency/readConcurrencyConfig.d.ts +60 -0
- package/dist/concurrency/readConcurrencyConfig.js +60 -0
- package/dist/concurrency/readConcurrencyConfig.js.map +1 -0
- package/dist/defineNode.d.ts +8 -0
- package/dist/defineNode.js +25 -5
- package/dist/defineNode.js.map +1 -1
- package/dist/graphql/GraphQLSchemaGenerator.js +1 -1
- package/dist/graphql/GraphQLSchemaGenerator.js.map +1 -1
- package/dist/idempotency/resolveIdempotencyKey.d.ts +20 -0
- package/dist/idempotency/resolveIdempotencyKey.js +37 -0
- package/dist/idempotency/resolveIdempotencyKey.js.map +1 -0
- package/dist/index.d.ts +30 -6
- package/dist/index.js +55 -6
- package/dist/index.js.map +1 -1
- package/dist/marketplace/RuntimeCatalog.d.ts +6 -0
- package/dist/marketplace/RuntimeCatalog.js.map +1 -1
- package/dist/marketplace/RuntimeDiscovery.d.ts +2 -2
- package/dist/marketplace/RuntimeDiscovery.js +18 -6
- package/dist/marketplace/RuntimeDiscovery.js.map +1 -1
- package/dist/monitoring/ConcurrencyMetrics.d.ts +82 -0
- package/dist/monitoring/ConcurrencyMetrics.js +139 -0
- package/dist/monitoring/ConcurrencyMetrics.js.map +1 -0
- package/dist/monitoring/ForEachWaitMetrics.d.ts +22 -0
- package/dist/monitoring/ForEachWaitMetrics.js +36 -0
- package/dist/monitoring/ForEachWaitMetrics.js.map +1 -0
- package/dist/monitoring/JanitorMetrics.d.ts +27 -0
- package/dist/monitoring/JanitorMetrics.js +48 -0
- package/dist/monitoring/JanitorMetrics.js.map +1 -0
- package/dist/openapi/OpenAPIGenerator.js +7 -2
- package/dist/openapi/OpenAPIGenerator.js.map +1 -1
- package/dist/runtime/PrimitiveStack.d.ts +64 -0
- package/dist/runtime/PrimitiveStack.js +92 -0
- package/dist/runtime/PrimitiveStack.js.map +1 -0
- package/dist/scheduling/DebounceBackend.d.ts +108 -0
- package/dist/scheduling/DebounceBackend.js +23 -0
- package/dist/scheduling/DebounceBackend.js.map +1 -0
- package/dist/scheduling/DebounceCoordinator.d.ts +141 -0
- package/dist/scheduling/DebounceCoordinator.js +362 -0
- package/dist/scheduling/DebounceCoordinator.js.map +1 -0
- package/dist/scheduling/DeferredDispatchSignal.d.ts +50 -0
- package/dist/scheduling/DeferredDispatchSignal.js +14 -0
- package/dist/scheduling/DeferredDispatchSignal.js.map +1 -0
- package/dist/scheduling/DeferredRunScheduler.d.ts +96 -0
- package/dist/scheduling/DeferredRunScheduler.js +256 -0
- package/dist/scheduling/DeferredRunScheduler.js.map +1 -0
- package/dist/scheduling/NatsKvDebounceBackend.d.ts +53 -0
- package/dist/scheduling/NatsKvDebounceBackend.js +334 -0
- package/dist/scheduling/NatsKvDebounceBackend.js.map +1 -0
- package/dist/scheduling/RedisDebounceBackend.d.ts +49 -0
- package/dist/scheduling/RedisDebounceBackend.js +356 -0
- package/dist/scheduling/RedisDebounceBackend.js.map +1 -0
- package/dist/scheduling/createDebounceBackend.d.ts +25 -0
- package/dist/scheduling/createDebounceBackend.js +39 -0
- package/dist/scheduling/createDebounceBackend.js.map +1 -0
- package/dist/scheduling/readSchedulingConfig.d.ts +24 -0
- package/dist/scheduling/readSchedulingConfig.js +52 -0
- package/dist/scheduling/readSchedulingConfig.js.map +1 -0
- package/dist/security/AuditLogger.js +1 -1
- package/dist/security/AuditLogger.js.map +1 -1
- package/dist/security/AuthMiddleware.d.ts +19 -20
- package/dist/security/AuthMiddleware.js +35 -20
- package/dist/security/AuthMiddleware.js.map +1 -1
- package/dist/security/OAuthProvider.js +2 -2
- package/dist/security/OAuthProvider.js.map +1 -1
- package/dist/security/SecretManager.js +14 -13
- package/dist/security/SecretManager.js.map +1 -1
- package/dist/security/index.d.ts +3 -1
- package/dist/security/index.js +3 -1
- package/dist/security/index.js.map +1 -1
- package/dist/testing/TestHarness.d.ts +27 -12
- package/dist/testing/TestHarness.js +19 -3
- package/dist/testing/TestHarness.js.map +1 -1
- package/dist/testing/WorkflowTestRunner.js +0 -7
- package/dist/testing/WorkflowTestRunner.js.map +1 -1
- package/dist/timeouts/StepTimeoutError.d.ts +22 -0
- package/dist/timeouts/StepTimeoutError.js +31 -0
- package/dist/timeouts/StepTimeoutError.js.map +1 -0
- package/dist/tracing/InMemoryRunStore.d.ts +41 -1
- package/dist/tracing/InMemoryRunStore.js +239 -0
- package/dist/tracing/InMemoryRunStore.js.map +1 -1
- package/dist/tracing/Janitor.d.ts +70 -0
- package/dist/tracing/Janitor.js +150 -0
- package/dist/tracing/Janitor.js.map +1 -0
- package/dist/tracing/PostgresRunStore.d.ts +57 -1
- package/dist/tracing/PostgresRunStore.js +711 -6
- package/dist/tracing/PostgresRunStore.js.map +1 -1
- package/dist/tracing/RoutingDiagnostics.d.ts +55 -0
- package/dist/tracing/RoutingDiagnostics.js +50 -0
- package/dist/tracing/RoutingDiagnostics.js.map +1 -0
- package/dist/tracing/RunStore.d.ts +181 -1
- package/dist/tracing/RunTracker.d.ts +244 -9
- package/dist/tracing/RunTracker.js +594 -1
- package/dist/tracing/RunTracker.js.map +1 -1
- package/dist/tracing/SqliteRunStore.d.ts +79 -2
- package/dist/tracing/SqliteRunStore.js +775 -16
- package/dist/tracing/SqliteRunStore.js.map +1 -1
- package/dist/tracing/TraceRouter.d.ts +20 -2
- package/dist/tracing/TraceRouter.js +612 -6
- package/dist/tracing/TraceRouter.js.map +1 -1
- package/dist/tracing/createStore.js +14 -3
- package/dist/tracing/createStore.js.map +1 -1
- package/dist/tracing/metadataFilter.d.ts +63 -0
- package/dist/tracing/metadataFilter.js +224 -0
- package/dist/tracing/metadataFilter.js.map +1 -0
- package/dist/tracing/sanitize.d.ts +11 -0
- package/dist/tracing/sanitize.js +29 -0
- package/dist/tracing/sanitize.js.map +1 -1
- package/dist/tracing/types.d.ts +672 -2
- package/dist/utils/createChildContext.d.ts +32 -0
- package/dist/utils/createChildContext.js +113 -0
- package/dist/utils/createChildContext.js.map +1 -0
- package/dist/utils/envAllowlist.d.ts +35 -0
- package/dist/utils/envAllowlist.js +113 -0
- package/dist/utils/envAllowlist.js.map +1 -0
- package/dist/version/RuntimeVersionValidator.d.ts +38 -0
- package/dist/version/RuntimeVersionValidator.js +121 -0
- package/dist/version/RuntimeVersionValidator.js.map +1 -0
- package/dist/visualization/WorkflowVisualizer.js +4 -4
- package/dist/visualization/WorkflowVisualizer.js.map +1 -1
- package/dist/workflow/PersistenceHelper.d.ts +18 -10
- package/dist/workflow/PersistenceHelper.js +35 -9
- package/dist/workflow/PersistenceHelper.js.map +1 -1
- package/dist/workflow/WorkflowNormalizer.d.ts +48 -42
- package/dist/workflow/WorkflowNormalizer.js +650 -18
- package/dist/workflow/WorkflowNormalizer.js.map +1 -1
- package/dist/workflow/WorkflowRegistry.d.ts +186 -0
- package/dist/workflow/WorkflowRegistry.js +202 -0
- package/dist/workflow/WorkflowRegistry.js.map +1 -0
- package/dist/workflow/sampleBody.d.ts +54 -0
- package/dist/workflow/sampleBody.js +320 -0
- package/dist/workflow/sampleBody.js.map +1 -0
- package/package.json +3 -8
- package/dist/adapters/HttpRuntimeAdapter.d.ts +0 -79
- package/dist/adapters/HttpRuntimeAdapter.js +0 -233
- package/dist/adapters/HttpRuntimeAdapter.js.map +0 -1
|
@@ -156,6 +156,172 @@ export class PostgresRunStore {
|
|
|
156
156
|
`);
|
|
157
157
|
},
|
|
158
158
|
},
|
|
159
|
+
{
|
|
160
|
+
// Tier 2 follow-up · durable schema for Tier 1 idempotency
|
|
161
|
+
// cache, Tier 2 #6 concurrency locks, and Tier 2 #5+#7
|
|
162
|
+
// scheduled dispatches. Previously these all delegated to
|
|
163
|
+
// the in-memory mirror only, so a process restart lost the
|
|
164
|
+
// state. The hybrid pattern (sync memory + async PG mirror)
|
|
165
|
+
// matches the existing workflow_runs/node_runs flow.
|
|
166
|
+
version: 3,
|
|
167
|
+
up: async () => {
|
|
168
|
+
await client.query(`
|
|
169
|
+
CREATE TABLE IF NOT EXISTS idempotency_cache (
|
|
170
|
+
workflow_name TEXT NOT NULL,
|
|
171
|
+
step_id TEXT NOT NULL,
|
|
172
|
+
idempotency_key TEXT NOT NULL,
|
|
173
|
+
data_json JSONB NOT NULL,
|
|
174
|
+
cached_at BIGINT NOT NULL,
|
|
175
|
+
expires_at BIGINT,
|
|
176
|
+
source_run_id TEXT NOT NULL,
|
|
177
|
+
source_node_run_id TEXT NOT NULL,
|
|
178
|
+
PRIMARY KEY (workflow_name, step_id, idempotency_key)
|
|
179
|
+
)
|
|
180
|
+
`);
|
|
181
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_idem_cache_expires ON idempotency_cache(expires_at)");
|
|
182
|
+
await client.query(`
|
|
183
|
+
CREATE TABLE IF NOT EXISTS concurrency_locks (
|
|
184
|
+
workflow_name TEXT NOT NULL,
|
|
185
|
+
concurrency_key TEXT NOT NULL,
|
|
186
|
+
run_id TEXT NOT NULL,
|
|
187
|
+
acquired_at BIGINT NOT NULL,
|
|
188
|
+
expires_at BIGINT NOT NULL,
|
|
189
|
+
PRIMARY KEY (workflow_name, concurrency_key, run_id)
|
|
190
|
+
)
|
|
191
|
+
`);
|
|
192
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_locks_expires ON concurrency_locks(expires_at)");
|
|
193
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_locks_workflow_key ON concurrency_locks(workflow_name, concurrency_key)");
|
|
194
|
+
await client.query(`
|
|
195
|
+
CREATE TABLE IF NOT EXISTS scheduled_dispatches (
|
|
196
|
+
run_id TEXT PRIMARY KEY,
|
|
197
|
+
workflow_name TEXT NOT NULL,
|
|
198
|
+
trigger_type TEXT NOT NULL,
|
|
199
|
+
scheduled_at BIGINT NOT NULL,
|
|
200
|
+
expires_at BIGINT,
|
|
201
|
+
dispatch_status TEXT NOT NULL,
|
|
202
|
+
payload_json JSONB NOT NULL,
|
|
203
|
+
created_at BIGINT NOT NULL
|
|
204
|
+
)
|
|
205
|
+
`);
|
|
206
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_scheduled_dispatches_at ON scheduled_dispatches(scheduled_at)");
|
|
207
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_scheduled_dispatches_trigger ON scheduled_dispatches(trigger_type, workflow_name)");
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
// PR 4 — wait.for / wait.until step primitive needs a
|
|
212
|
+
// resume cursor so dispatchDeferred re-entry skips
|
|
213
|
+
// already-completed pre-wait steps. Mirror sqlite v10.
|
|
214
|
+
version: 4,
|
|
215
|
+
up: async () => {
|
|
216
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS last_completed_step_index INTEGER");
|
|
217
|
+
},
|
|
218
|
+
},
|
|
219
|
+
{
|
|
220
|
+
// PR 1-5 polish · catch up to sqlite's workflow_runs columns.
|
|
221
|
+
// PG migrations historically lagged: v3 added the cache /
|
|
222
|
+
// locks / dispatches tables and v4 added
|
|
223
|
+
// `last_completed_step_index`, but the columns sqlite added
|
|
224
|
+
// across migrations 3 / 5 / 6 / 8 (`environment`,
|
|
225
|
+
// `replay_of`, `parent_run_id`, `parent_node_run_id`,
|
|
226
|
+
// `scheduled_at`, `expires_at`, `debounce_key`,
|
|
227
|
+
// `debounce_mode`, `ping_count`) were never mirrored.
|
|
228
|
+
// `saveRun`/`updateRun`/`rowToRun` silently dropped them
|
|
229
|
+
// across restarts, so PG-backed deployments lost replay
|
|
230
|
+
// lineage, sub-workflow lineage, and ALL Tier 2 #5+#7
|
|
231
|
+
// scheduling state on every restart. This migration adds
|
|
232
|
+
// the columns + indexes so PG matches sqlite. Pre-existing
|
|
233
|
+
// rows get NULL on every new column (backward-compat;
|
|
234
|
+
// `rowToRun` reads NULL `environment` as "production" to
|
|
235
|
+
// match sqlite's legacy default).
|
|
236
|
+
version: 5,
|
|
237
|
+
up: async () => {
|
|
238
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS environment TEXT");
|
|
239
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS replay_of TEXT");
|
|
240
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS parent_run_id TEXT");
|
|
241
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS parent_node_run_id TEXT");
|
|
242
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS scheduled_at BIGINT");
|
|
243
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS expires_at BIGINT");
|
|
244
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS debounce_key TEXT");
|
|
245
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS debounce_mode TEXT");
|
|
246
|
+
await client.query("ALTER TABLE workflow_runs ADD COLUMN IF NOT EXISTS ping_count INTEGER");
|
|
247
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_runs_environment ON workflow_runs(environment)");
|
|
248
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_runs_replay_of ON workflow_runs(replay_of)");
|
|
249
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_runs_parent_run ON workflow_runs(parent_run_id)");
|
|
250
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_runs_scheduled_at ON workflow_runs(scheduled_at)");
|
|
251
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_runs_debounce_key ON workflow_runs(workflow_name, debounce_key)");
|
|
252
|
+
},
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
// Tier C #2 — cross-process scheduler coordination.
|
|
256
|
+
//
|
|
257
|
+
// Multi-process PG deployments risk double-firing the same
|
|
258
|
+
// dispatch when both processes' `recoverDispatches()`
|
|
259
|
+
// register timers against the same row. The claim columns
|
|
260
|
+
// let processes atomically take ownership; the
|
|
261
|
+
// `DeferredRunScheduler` heartbeats `claimed_at` while a
|
|
262
|
+
// timer is registered; a dead process's claim expires
|
|
263
|
+
// after `BLOK_SCHEDULER_CLAIM_LEASE_MS` and a surviving
|
|
264
|
+
// process can take over on its next recovery.
|
|
265
|
+
version: 6,
|
|
266
|
+
up: async () => {
|
|
267
|
+
await client.query("ALTER TABLE scheduled_dispatches ADD COLUMN IF NOT EXISTS claimed_by TEXT");
|
|
268
|
+
await client.query("ALTER TABLE scheduled_dispatches ADD COLUMN IF NOT EXISTS claimed_at BIGINT");
|
|
269
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_scheduled_dispatches_claim ON scheduled_dispatches(claimed_by, claimed_at)");
|
|
270
|
+
},
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
// E2 · server-side saved filters for the runs list.
|
|
274
|
+
// Matches the sqlite store's v14 schema. Hybrid pattern:
|
|
275
|
+
// rehydrated into the in-memory mirror on boot via
|
|
276
|
+
// `loadRecent`, mutated through the same `enqueueWrite`
|
|
277
|
+
// fan-out used by dashboards.
|
|
278
|
+
version: 7,
|
|
279
|
+
up: async () => {
|
|
280
|
+
await client.query(`
|
|
281
|
+
CREATE TABLE IF NOT EXISTS trace_saved_filters (
|
|
282
|
+
id TEXT PRIMARY KEY,
|
|
283
|
+
name TEXT NOT NULL UNIQUE,
|
|
284
|
+
status TEXT NOT NULL DEFAULT '',
|
|
285
|
+
tags_input TEXT NOT NULL DEFAULT '',
|
|
286
|
+
metadata_input TEXT NOT NULL DEFAULT '',
|
|
287
|
+
created_at BIGINT NOT NULL,
|
|
288
|
+
updated_at BIGINT NOT NULL
|
|
289
|
+
)
|
|
290
|
+
`);
|
|
291
|
+
await client.query("CREATE INDEX IF NOT EXISTS idx_saved_filters_updated_at ON trace_saved_filters(updated_at)");
|
|
292
|
+
},
|
|
293
|
+
},
|
|
294
|
+
{
|
|
295
|
+
// v0.6 follow-up to #100 — recorded sample bodies for the
|
|
296
|
+
// Studio empty-state curl. Matches the sqlite store's v15
|
|
297
|
+
// schema. One row per workflow (PK on workflow_name);
|
|
298
|
+
// the trigger only writes the FIRST successful run's body
|
|
299
|
+
// so the operator-visible curl stays stable.
|
|
300
|
+
version: 8,
|
|
301
|
+
up: async () => {
|
|
302
|
+
await client.query(`
|
|
303
|
+
CREATE TABLE IF NOT EXISTS trace_workflow_samples (
|
|
304
|
+
workflow_name TEXT PRIMARY KEY,
|
|
305
|
+
body_json JSONB NOT NULL,
|
|
306
|
+
source_run_id TEXT NOT NULL,
|
|
307
|
+
recorded_at BIGINT NOT NULL
|
|
308
|
+
)
|
|
309
|
+
`);
|
|
310
|
+
},
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
// v0.6 follow-up — bag column for Studio-visible step flags
|
|
314
|
+
// (`wait`, `dispatch`, `subworkflowDepth`, `middleware`,
|
|
315
|
+
// `iterationIndex`). Mirrors sqlite v16. Previously these
|
|
316
|
+
// fields lived only on the in-memory NodeRun and vanished
|
|
317
|
+
// on round-trip — rail badges disappeared after process
|
|
318
|
+
// restart. JSONB so future flag additions don't need new
|
|
319
|
+
// migrations.
|
|
320
|
+
version: 9,
|
|
321
|
+
up: async () => {
|
|
322
|
+
await client.query("ALTER TABLE node_runs ADD COLUMN IF NOT EXISTS flags_json JSONB");
|
|
323
|
+
},
|
|
324
|
+
},
|
|
159
325
|
];
|
|
160
326
|
for (const m of migrations) {
|
|
161
327
|
if (!applied.has(m.version)) {
|
|
@@ -176,6 +342,20 @@ export class PostgresRunStore {
|
|
|
176
342
|
client.release();
|
|
177
343
|
}
|
|
178
344
|
}
|
|
345
|
+
/**
|
|
346
|
+
* Review fix-up · CONCERN-2. Per-table cap on `loadRecent`'s rehydration
|
|
347
|
+
* queries. Without it, a deployment with 1M+ idempotency cache entries
|
|
348
|
+
* spends seconds + memory on every boot loading rows it'll never read
|
|
349
|
+
* before the Janitor sweeps them. Default 100K rows per table — well
|
|
350
|
+
* above any reasonable hot-set size yet bounded for boot latency.
|
|
351
|
+
*/
|
|
352
|
+
getLoadRecentLimit() {
|
|
353
|
+
const raw = process.env.BLOK_PG_LOADRECENT_LIMIT;
|
|
354
|
+
if (!raw || !/^\d+$/.test(raw))
|
|
355
|
+
return 100_000;
|
|
356
|
+
const n = Number(raw);
|
|
357
|
+
return n > 0 ? n : 100_000;
|
|
358
|
+
}
|
|
179
359
|
async loadRecent() {
|
|
180
360
|
const client = await this.pool.connect();
|
|
181
361
|
try {
|
|
@@ -207,6 +387,130 @@ export class PostgresRunStore {
|
|
|
207
387
|
for (const row of dashRows) {
|
|
208
388
|
this.memory.saveDashboard(this.rowToDashboard(row));
|
|
209
389
|
}
|
|
390
|
+
// E2 · rehydrate saved filters. Unbounded (operator-curated
|
|
391
|
+
// set, not a per-run table — typical fleet has < 100 entries).
|
|
392
|
+
try {
|
|
393
|
+
const { rows: filterRows } = await client.query("SELECT * FROM trace_saved_filters ORDER BY updated_at DESC");
|
|
394
|
+
for (const row of filterRows) {
|
|
395
|
+
this.memory.upsertSavedFilter({
|
|
396
|
+
id: row.id,
|
|
397
|
+
name: row.name,
|
|
398
|
+
status: row.status,
|
|
399
|
+
tagsInput: row.tags_input,
|
|
400
|
+
metadataInput: row.metadata_input,
|
|
401
|
+
createdAt: Number(row.created_at),
|
|
402
|
+
updatedAt: Number(row.updated_at),
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
// First-time deployments may not have run the v7 migration
|
|
408
|
+
// when this loadRecent fires (the migration is at boot;
|
|
409
|
+
// this should follow). Log + continue.
|
|
410
|
+
console.warn("[blok][pg] failed to load saved filters:", err.message);
|
|
411
|
+
}
|
|
412
|
+
// v0.6 · rehydrate recorded sample bodies. Cardinality
|
|
413
|
+
// bounded by the number of workflows in the deployment, not
|
|
414
|
+
// the run volume — unbounded SELECT is fine.
|
|
415
|
+
try {
|
|
416
|
+
const { rows: sampleRows } = await client.query("SELECT * FROM trace_workflow_samples");
|
|
417
|
+
for (const row of sampleRows) {
|
|
418
|
+
this.memory.recordWorkflowSample({
|
|
419
|
+
workflowName: row.workflow_name,
|
|
420
|
+
body: typeof row.body_json === "string" ? JSON.parse(row.body_json) : row.body_json,
|
|
421
|
+
sourceRunId: row.source_run_id,
|
|
422
|
+
recordedAt: Number(row.recorded_at),
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
catch (err) {
|
|
427
|
+
console.warn("[blok][pg] failed to load workflow samples:", err.message);
|
|
428
|
+
}
|
|
429
|
+
// Tier 2 follow-up · rehydrate idempotency cache (un-expired entries only).
|
|
430
|
+
//
|
|
431
|
+
// Review fix-up · CONCERN-2. Add `ORDER BY cached_at DESC LIMIT N`.
|
|
432
|
+
// Newest cache entries are most likely to be re-hit by the next
|
|
433
|
+
// idempotent step; older ones the Janitor will sweep on its next
|
|
434
|
+
// pass. Without this LIMIT, deployments with 1M+ rows OOM at boot.
|
|
435
|
+
const now = Date.now();
|
|
436
|
+
const loadRecentLimit = this.getLoadRecentLimit();
|
|
437
|
+
try {
|
|
438
|
+
const { rows: idemRows } = await client.query(`SELECT * FROM idempotency_cache
|
|
439
|
+
WHERE expires_at IS NULL OR expires_at > $1
|
|
440
|
+
ORDER BY cached_at DESC
|
|
441
|
+
LIMIT $2`, [now, loadRecentLimit]);
|
|
442
|
+
for (const row of idemRows) {
|
|
443
|
+
this.memory.setIdempotencyCache(row.workflow_name, row.step_id, row.idempotency_key, {
|
|
444
|
+
data: typeof row.data_json === "string" ? JSON.parse(row.data_json) : row.data_json,
|
|
445
|
+
cachedAt: Number(row.cached_at),
|
|
446
|
+
expiresAt: row.expires_at !== null ? Number(row.expires_at) : null,
|
|
447
|
+
sourceRunId: row.source_run_id,
|
|
448
|
+
sourceNodeRunId: row.source_node_run_id,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
catch (err) {
|
|
453
|
+
// Pre-v3 PG schema may not have the table yet — fall through quietly.
|
|
454
|
+
if (!String(err.message).match(/relation .* does not exist/i)) {
|
|
455
|
+
console.error("[PostgresRunStore] idempotency_cache load failed:", err.message);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
// Tier 2 follow-up · rehydrate concurrency leases (un-expired only).
|
|
459
|
+
//
|
|
460
|
+
// Review fix-up · CONCERN-2. Add `ORDER BY expires_at DESC LIMIT N`.
|
|
461
|
+
// Locks with the longest remaining lease are most likely still
|
|
462
|
+
// active and worth restoring; expired ones are filtered already.
|
|
463
|
+
try {
|
|
464
|
+
const { rows: lockRows } = await client.query(`SELECT * FROM concurrency_locks
|
|
465
|
+
WHERE expires_at > $1
|
|
466
|
+
ORDER BY expires_at DESC
|
|
467
|
+
LIMIT $2`, [now, loadRecentLimit]);
|
|
468
|
+
for (const row of lockRows) {
|
|
469
|
+
this.memory.acquireConcurrencySlot(row.workflow_name, row.concurrency_key, Number.MAX_SAFE_INTEGER, // skip the limit check — we're restoring, not granting
|
|
470
|
+
row.run_id, Number(row.expires_at));
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
catch (err) {
|
|
474
|
+
if (!String(err.message).match(/relation .* does not exist/i)) {
|
|
475
|
+
console.error("[PostgresRunStore] concurrency_locks load failed:", err.message);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
// Tier 2 follow-up · rehydrate scheduled dispatches.
|
|
479
|
+
// PR 2 A5 — ORDER BY scheduled_at ASC so past-due dispatches
|
|
480
|
+
// hydrate first. recoverDispatches's past-due → fire-immediately
|
|
481
|
+
// path benefits from the ordering.
|
|
482
|
+
//
|
|
483
|
+
// Review fix-up · CONCERN-2. Add LIMIT for defense-in-depth.
|
|
484
|
+
// Even with the 1MB payload cap (PR 2 A4), 100K dispatches × 1MB
|
|
485
|
+
// = 100GB of JSON-decode work at boot. Past-due first means the
|
|
486
|
+
// LIMIT caps how many recover per boot; the rest surface on the
|
|
487
|
+
// Janitor's next sweep or a subsequent boot.
|
|
488
|
+
try {
|
|
489
|
+
const { rows: dispatchRows } = await client.query("SELECT * FROM scheduled_dispatches ORDER BY scheduled_at ASC LIMIT $1", [loadRecentLimit]);
|
|
490
|
+
for (const row of dispatchRows) {
|
|
491
|
+
this.memory.upsertScheduledDispatch({
|
|
492
|
+
runId: row.run_id,
|
|
493
|
+
workflowName: row.workflow_name,
|
|
494
|
+
triggerType: row.trigger_type,
|
|
495
|
+
scheduledAt: Number(row.scheduled_at),
|
|
496
|
+
expiresAt: row.expires_at !== null ? Number(row.expires_at) : undefined,
|
|
497
|
+
dispatchStatus: row.dispatch_status,
|
|
498
|
+
payload: typeof row.payload_json === "string" ? JSON.parse(row.payload_json) : row.payload_json,
|
|
499
|
+
createdAt: Number(row.created_at),
|
|
500
|
+
// Tier C #2 — surface claim state across restart so
|
|
501
|
+
// the trigger doesn't re-claim a row another process
|
|
502
|
+
// already owns. PG columns may not exist on pre-v6
|
|
503
|
+
// databases — guard with `?? undefined`.
|
|
504
|
+
claimedBy: row.claimed_by !== null && row.claimed_by !== undefined ? row.claimed_by : undefined,
|
|
505
|
+
claimedAt: row.claimed_at !== null && row.claimed_at !== undefined ? Number(row.claimed_at) : undefined,
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
catch (err) {
|
|
510
|
+
if (!String(err.message).match(/relation .* does not exist/i)) {
|
|
511
|
+
console.error("[PostgresRunStore] scheduled_dispatches load failed:", err.message);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
210
514
|
}
|
|
211
515
|
finally {
|
|
212
516
|
client.release();
|
|
@@ -215,12 +519,22 @@ export class PostgresRunStore {
|
|
|
215
519
|
// === Writes (sync via memory + async queue to PG) ===
|
|
216
520
|
saveRun(run) {
|
|
217
521
|
this.memory.saveRun(run);
|
|
522
|
+
// PR 1-5 polish · column set mirrors sqlite saveRun (24 columns).
|
|
523
|
+
// PG migration v5 added the trailing 9 (environment / replay_of /
|
|
524
|
+
// parent_run_id / parent_node_run_id / scheduled_at / expires_at /
|
|
525
|
+
// debounce_key / debounce_mode / ping_count); PG migration v4 added
|
|
526
|
+
// last_completed_step_index. Without these the PG mirror silently
|
|
527
|
+
// dropped scheduling + lineage + resume-cursor state across restart.
|
|
218
528
|
this.enqueueWrite(() => this.pool
|
|
219
529
|
.query(`INSERT INTO workflow_runs
|
|
220
530
|
(id, workflow_name, workflow_path, trigger_type, trigger_summary,
|
|
221
531
|
status, started_at, finished_at, duration_ms, error_json,
|
|
222
|
-
tags_json, metadata_json, node_count, completed_nodes
|
|
223
|
-
|
|
532
|
+
tags_json, metadata_json, node_count, completed_nodes,
|
|
533
|
+
environment, replay_of, parent_run_id, parent_node_run_id,
|
|
534
|
+
scheduled_at, expires_at, debounce_key, debounce_mode,
|
|
535
|
+
ping_count, last_completed_step_index)
|
|
536
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
|
|
537
|
+
$15, $16, $17, $18, $19, $20, $21, $22, $23, $24)
|
|
224
538
|
ON CONFLICT (id) DO UPDATE SET
|
|
225
539
|
status = EXCLUDED.status,
|
|
226
540
|
finished_at = EXCLUDED.finished_at,
|
|
@@ -228,7 +542,17 @@ export class PostgresRunStore {
|
|
|
228
542
|
error_json = EXCLUDED.error_json,
|
|
229
543
|
tags_json = EXCLUDED.tags_json,
|
|
230
544
|
metadata_json = EXCLUDED.metadata_json,
|
|
231
|
-
completed_nodes = EXCLUDED.completed_nodes
|
|
545
|
+
completed_nodes = EXCLUDED.completed_nodes,
|
|
546
|
+
environment = EXCLUDED.environment,
|
|
547
|
+
replay_of = EXCLUDED.replay_of,
|
|
548
|
+
parent_run_id = EXCLUDED.parent_run_id,
|
|
549
|
+
parent_node_run_id = EXCLUDED.parent_node_run_id,
|
|
550
|
+
scheduled_at = EXCLUDED.scheduled_at,
|
|
551
|
+
expires_at = EXCLUDED.expires_at,
|
|
552
|
+
debounce_key = EXCLUDED.debounce_key,
|
|
553
|
+
debounce_mode = EXCLUDED.debounce_mode,
|
|
554
|
+
ping_count = EXCLUDED.ping_count,
|
|
555
|
+
last_completed_step_index = EXCLUDED.last_completed_step_index`, [
|
|
232
556
|
run.id,
|
|
233
557
|
run.workflowName,
|
|
234
558
|
run.workflowPath,
|
|
@@ -243,6 +567,16 @@ export class PostgresRunStore {
|
|
|
243
567
|
run.metadata ? JSON.stringify(run.metadata) : null,
|
|
244
568
|
run.nodeCount,
|
|
245
569
|
run.completedNodes,
|
|
570
|
+
run.environment ?? null,
|
|
571
|
+
run.replayOf ?? null,
|
|
572
|
+
run.parentRunId ?? null,
|
|
573
|
+
run.parentNodeRunId ?? null,
|
|
574
|
+
run.scheduledAt ?? null,
|
|
575
|
+
run.expiresAt ?? null,
|
|
576
|
+
run.debounceKey ?? null,
|
|
577
|
+
run.debounceMode ?? null,
|
|
578
|
+
run.pingCount ?? null,
|
|
579
|
+
run.lastCompletedStepIndex ?? null,
|
|
246
580
|
])
|
|
247
581
|
.then(() => { }));
|
|
248
582
|
}
|
|
@@ -279,6 +613,55 @@ export class PostgresRunStore {
|
|
|
279
613
|
setClauses.push(`metadata_json = $${paramIdx++}`);
|
|
280
614
|
values.push(JSON.stringify(updates.metadata));
|
|
281
615
|
}
|
|
616
|
+
// PR 1-5 polish · scheduling, lineage, and resume-cursor fields
|
|
617
|
+
// updateRun must mirror sqlite. Each marker tracker method
|
|
618
|
+
// (markRunDelayed / markRunQueued / markRunDebounced / markRunExpired
|
|
619
|
+
// / recordDebouncePing / transitionRunToRunning / RunnerSteps wait
|
|
620
|
+
// branch) updates one or more of these. Without these clauses the
|
|
621
|
+
// in-memory mirror has the value but PG keeps the original (or
|
|
622
|
+
// NULL).
|
|
623
|
+
if (updates.replayOf !== undefined) {
|
|
624
|
+
setClauses.push(`replay_of = $${paramIdx++}`);
|
|
625
|
+
values.push(updates.replayOf);
|
|
626
|
+
}
|
|
627
|
+
if (updates.parentRunId !== undefined) {
|
|
628
|
+
setClauses.push(`parent_run_id = $${paramIdx++}`);
|
|
629
|
+
values.push(updates.parentRunId);
|
|
630
|
+
}
|
|
631
|
+
if (updates.parentNodeRunId !== undefined) {
|
|
632
|
+
setClauses.push(`parent_node_run_id = $${paramIdx++}`);
|
|
633
|
+
values.push(updates.parentNodeRunId);
|
|
634
|
+
}
|
|
635
|
+
if (updates.scheduledAt !== undefined) {
|
|
636
|
+
setClauses.push(`scheduled_at = $${paramIdx++}`);
|
|
637
|
+
values.push(updates.scheduledAt);
|
|
638
|
+
}
|
|
639
|
+
if (updates.expiresAt !== undefined) {
|
|
640
|
+
setClauses.push(`expires_at = $${paramIdx++}`);
|
|
641
|
+
values.push(updates.expiresAt);
|
|
642
|
+
}
|
|
643
|
+
if (updates.debounceKey !== undefined) {
|
|
644
|
+
setClauses.push(`debounce_key = $${paramIdx++}`);
|
|
645
|
+
values.push(updates.debounceKey);
|
|
646
|
+
}
|
|
647
|
+
if (updates.debounceMode !== undefined) {
|
|
648
|
+
setClauses.push(`debounce_mode = $${paramIdx++}`);
|
|
649
|
+
values.push(updates.debounceMode);
|
|
650
|
+
}
|
|
651
|
+
if (updates.pingCount !== undefined) {
|
|
652
|
+
setClauses.push(`ping_count = $${paramIdx++}`);
|
|
653
|
+
values.push(updates.pingCount);
|
|
654
|
+
}
|
|
655
|
+
if (updates.lastCompletedStepIndex !== undefined) {
|
|
656
|
+
setClauses.push(`last_completed_step_index = $${paramIdx++}`);
|
|
657
|
+
values.push(updates.lastCompletedStepIndex);
|
|
658
|
+
}
|
|
659
|
+
// `transitionRunToRunning` (Tier 2 #5+#7) preserves the original
|
|
660
|
+
// startedAt by updating it. Mirror sqlite, which also accepts it.
|
|
661
|
+
if (updates.startedAt !== undefined) {
|
|
662
|
+
setClauses.push(`started_at = $${paramIdx++}`);
|
|
663
|
+
values.push(updates.startedAt);
|
|
664
|
+
}
|
|
282
665
|
if (setClauses.length === 0)
|
|
283
666
|
return;
|
|
284
667
|
values.push(runId);
|
|
@@ -288,20 +671,22 @@ export class PostgresRunStore {
|
|
|
288
671
|
}
|
|
289
672
|
saveNodeRun(nodeRun) {
|
|
290
673
|
this.memory.saveNodeRun(nodeRun);
|
|
674
|
+
const flagsJson = encodeNodeRunFlagsForPg(nodeRun);
|
|
291
675
|
this.enqueueWrite(() => this.pool
|
|
292
676
|
.query(`INSERT INTO node_runs
|
|
293
677
|
(id, run_id, node_name, node_type, runtime_kind,
|
|
294
678
|
status, started_at, finished_at, duration_ms,
|
|
295
679
|
inputs_json, outputs_json, error_json,
|
|
296
|
-
parent_node_id, depth, step_index, metrics_json)
|
|
297
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
|
680
|
+
parent_node_id, depth, step_index, metrics_json, flags_json)
|
|
681
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
|
298
682
|
ON CONFLICT (id) DO UPDATE SET
|
|
299
683
|
status = EXCLUDED.status,
|
|
300
684
|
finished_at = EXCLUDED.finished_at,
|
|
301
685
|
duration_ms = EXCLUDED.duration_ms,
|
|
302
686
|
outputs_json = EXCLUDED.outputs_json,
|
|
303
687
|
error_json = EXCLUDED.error_json,
|
|
304
|
-
metrics_json = EXCLUDED.metrics_json
|
|
688
|
+
metrics_json = EXCLUDED.metrics_json,
|
|
689
|
+
flags_json = EXCLUDED.flags_json`, [
|
|
305
690
|
nodeRun.id,
|
|
306
691
|
nodeRun.runId,
|
|
307
692
|
nodeRun.nodeName,
|
|
@@ -318,6 +703,7 @@ export class PostgresRunStore {
|
|
|
318
703
|
nodeRun.depth,
|
|
319
704
|
nodeRun.stepIndex,
|
|
320
705
|
nodeRun.metrics ? JSON.stringify(nodeRun.metrics) : null,
|
|
706
|
+
flagsJson,
|
|
321
707
|
])
|
|
322
708
|
.then(() => { }));
|
|
323
709
|
}
|
|
@@ -402,6 +788,12 @@ export class PostgresRunStore {
|
|
|
402
788
|
getNodeRun(nodeRunId) {
|
|
403
789
|
return this.memory.getNodeRun(nodeRunId);
|
|
404
790
|
}
|
|
791
|
+
getRunsByParent(parentRunId) {
|
|
792
|
+
// Tier 2 sub-workflow lineage. Same in-memory delegation strategy
|
|
793
|
+
// as the idempotency cache — durable PG schema for parent_run_id
|
|
794
|
+
// is deferred to a follow-up.
|
|
795
|
+
return this.memory.getRunsByParent(parentRunId);
|
|
796
|
+
}
|
|
405
797
|
getEvents(runId, since) {
|
|
406
798
|
return this.memory.getEvents(runId, since);
|
|
407
799
|
}
|
|
@@ -480,6 +872,62 @@ export class PostgresRunStore {
|
|
|
480
872
|
values.push(dashboardId);
|
|
481
873
|
this.enqueueWrite(() => this.pool.query(`UPDATE dashboards SET ${setClauses.join(", ")} WHERE id = $${paramIdx}`, values).then(() => { }));
|
|
482
874
|
}
|
|
875
|
+
// === Saved filters (E2) ===
|
|
876
|
+
upsertSavedFilter(filter) {
|
|
877
|
+
const persisted = this.memory.upsertSavedFilter(filter);
|
|
878
|
+
this.enqueueWrite(() => this.pool
|
|
879
|
+
.query(`INSERT INTO trace_saved_filters
|
|
880
|
+
(id, name, status, tags_input, metadata_input, created_at, updated_at)
|
|
881
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
882
|
+
ON CONFLICT (name) DO UPDATE SET
|
|
883
|
+
status = EXCLUDED.status,
|
|
884
|
+
tags_input = EXCLUDED.tags_input,
|
|
885
|
+
metadata_input = EXCLUDED.metadata_input,
|
|
886
|
+
updated_at = EXCLUDED.updated_at`, [
|
|
887
|
+
persisted.id,
|
|
888
|
+
persisted.name,
|
|
889
|
+
persisted.status,
|
|
890
|
+
persisted.tagsInput,
|
|
891
|
+
persisted.metadataInput,
|
|
892
|
+
persisted.createdAt,
|
|
893
|
+
persisted.updatedAt,
|
|
894
|
+
])
|
|
895
|
+
.then(() => { }));
|
|
896
|
+
return persisted;
|
|
897
|
+
}
|
|
898
|
+
listSavedFilters() {
|
|
899
|
+
return this.memory.listSavedFilters();
|
|
900
|
+
}
|
|
901
|
+
deleteSavedFilter(name) {
|
|
902
|
+
const removed = this.memory.deleteSavedFilter(name);
|
|
903
|
+
if (removed) {
|
|
904
|
+
this.enqueueWrite(() => this.pool.query("DELETE FROM trace_saved_filters WHERE name = $1", [name]).then(() => { }));
|
|
905
|
+
}
|
|
906
|
+
return removed;
|
|
907
|
+
}
|
|
908
|
+
// === Sample-body recording (option C) ===
|
|
909
|
+
recordWorkflowSample(sample) {
|
|
910
|
+
// First-record-wins. Memory holds the canonical sample (sync
|
|
911
|
+
// reads). PG INSERT uses ON CONFLICT DO NOTHING — same
|
|
912
|
+
// "first-record sticks" semantic as the sqlite store.
|
|
913
|
+
const persisted = this.memory.recordWorkflowSample(sample);
|
|
914
|
+
this.enqueueWrite(() => this.pool
|
|
915
|
+
.query(`INSERT INTO trace_workflow_samples (workflow_name, body_json, source_run_id, recorded_at)
|
|
916
|
+
VALUES ($1, $2, $3, $4)
|
|
917
|
+
ON CONFLICT (workflow_name) DO NOTHING`, [persisted.workflowName, JSON.stringify(persisted.body), persisted.sourceRunId, persisted.recordedAt])
|
|
918
|
+
.then(() => { }));
|
|
919
|
+
return persisted;
|
|
920
|
+
}
|
|
921
|
+
getWorkflowSample(workflowName) {
|
|
922
|
+
return this.memory.getWorkflowSample(workflowName);
|
|
923
|
+
}
|
|
924
|
+
deleteWorkflowSample(workflowName) {
|
|
925
|
+
const removed = this.memory.deleteWorkflowSample(workflowName);
|
|
926
|
+
if (removed) {
|
|
927
|
+
this.enqueueWrite(() => this.pool.query("DELETE FROM trace_workflow_samples WHERE workflow_name = $1", [workflowName]).then(() => { }));
|
|
928
|
+
}
|
|
929
|
+
return removed;
|
|
930
|
+
}
|
|
483
931
|
// === Cleanup ===
|
|
484
932
|
clearAll() {
|
|
485
933
|
const count = this.memory.clearAll();
|
|
@@ -489,6 +937,23 @@ export class PostgresRunStore {
|
|
|
489
937
|
await this.pool.query("DELETE FROM node_runs");
|
|
490
938
|
await this.pool.query("DELETE FROM workflow_runs");
|
|
491
939
|
await this.pool.query("DELETE FROM dashboards");
|
|
940
|
+
// Tier 2 follow-up — wipe durable tables too. Wrapped in
|
|
941
|
+
// individual try blocks so a missing table (pre-v3 schema) on
|
|
942
|
+
// one doesn't abort the others.
|
|
943
|
+
for (const sql of [
|
|
944
|
+
"DELETE FROM idempotency_cache",
|
|
945
|
+
"DELETE FROM concurrency_locks",
|
|
946
|
+
"DELETE FROM scheduled_dispatches",
|
|
947
|
+
]) {
|
|
948
|
+
try {
|
|
949
|
+
await this.pool.query(sql);
|
|
950
|
+
}
|
|
951
|
+
catch (err) {
|
|
952
|
+
if (!String(err.message).match(/relation .* does not exist/i)) {
|
|
953
|
+
console.error(`[PostgresRunStore] ${sql} failed:`, err.message);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
}
|
|
492
957
|
});
|
|
493
958
|
return count;
|
|
494
959
|
}
|
|
@@ -513,6 +978,204 @@ export class PostgresRunStore {
|
|
|
513
978
|
});
|
|
514
979
|
this.closed = true;
|
|
515
980
|
}
|
|
981
|
+
// === Idempotency cache (Tier 1) ===
|
|
982
|
+
//
|
|
983
|
+
// PG durable cache is out of scope for this PR — delegate to the
|
|
984
|
+
// in-memory layer this store already uses for hot reads. Same-process
|
|
985
|
+
// hits work exactly like the SQLite store; cross-process / cross-restart
|
|
986
|
+
// hits are deferred to a follow-up PG schema migration. Operators
|
|
987
|
+
// running PG today retain pre-Phase-3 behaviour (no caching) on a fresh
|
|
988
|
+
// process and gain in-memory caching within a single process lifetime.
|
|
989
|
+
//
|
|
990
|
+
// Tier 2 follow-up (migration v3) — sync reads stay on the in-memory
|
|
991
|
+
// mirror; writes async-persist to PG. On boot, `loadRecent()` rehydrates
|
|
992
|
+
// the in-memory cache from PG so deferred dispatches + idempotency
|
|
993
|
+
// entries + concurrency leases survive restarts.
|
|
994
|
+
getIdempotencyCache(workflowName, stepId, key) {
|
|
995
|
+
return this.memory.getIdempotencyCache(workflowName, stepId, key);
|
|
996
|
+
}
|
|
997
|
+
setIdempotencyCache(workflowName, stepId, key, entry) {
|
|
998
|
+
this.memory.setIdempotencyCache(workflowName, stepId, key, entry);
|
|
999
|
+
this.enqueueWrite(() => this.pool
|
|
1000
|
+
.query(`INSERT INTO idempotency_cache
|
|
1001
|
+
(workflow_name, step_id, idempotency_key, data_json,
|
|
1002
|
+
cached_at, expires_at, source_run_id, source_node_run_id)
|
|
1003
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
1004
|
+
ON CONFLICT (workflow_name, step_id, idempotency_key) DO UPDATE SET
|
|
1005
|
+
data_json = EXCLUDED.data_json,
|
|
1006
|
+
cached_at = EXCLUDED.cached_at,
|
|
1007
|
+
expires_at = EXCLUDED.expires_at,
|
|
1008
|
+
source_run_id = EXCLUDED.source_run_id,
|
|
1009
|
+
source_node_run_id = EXCLUDED.source_node_run_id`, [
|
|
1010
|
+
workflowName,
|
|
1011
|
+
stepId,
|
|
1012
|
+
key,
|
|
1013
|
+
JSON.stringify(entry.data),
|
|
1014
|
+
entry.cachedAt,
|
|
1015
|
+
entry.expiresAt,
|
|
1016
|
+
entry.sourceRunId,
|
|
1017
|
+
entry.sourceNodeRunId,
|
|
1018
|
+
])
|
|
1019
|
+
.then(() => { }));
|
|
1020
|
+
}
|
|
1021
|
+
purgeExpiredIdempotencyCache(now) {
|
|
1022
|
+
const removed = this.memory.purgeExpiredIdempotencyCache(now);
|
|
1023
|
+
this.enqueueWrite(() => this.pool
|
|
1024
|
+
.query("DELETE FROM idempotency_cache WHERE expires_at IS NOT NULL AND expires_at <= $1", [now])
|
|
1025
|
+
.then(() => { }));
|
|
1026
|
+
return removed;
|
|
1027
|
+
}
|
|
1028
|
+
// === Concurrency gating (Tier 2 #6) ===
|
|
1029
|
+
// Sync grants happen on the in-memory mirror; PG mirror is async-only.
|
|
1030
|
+
// Cross-process coordination via the gate itself requires the dedicated
|
|
1031
|
+
// `BLOK_CONCURRENCY_BACKEND=nats-kv` backend (Tier 2 #6 follow-up).
|
|
1032
|
+
// PG persistence here is purely for crash-recovery — boot loads active
|
|
1033
|
+
// (un-expired) leases back into memory so a process restart doesn't
|
|
1034
|
+
// over-grant.
|
|
1035
|
+
acquireConcurrencySlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt) {
|
|
1036
|
+
const result = this.memory.acquireConcurrencySlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt);
|
|
1037
|
+
if (result.acquired) {
|
|
1038
|
+
this.enqueueWrite(() => this.pool
|
|
1039
|
+
.query(`INSERT INTO concurrency_locks
|
|
1040
|
+
(workflow_name, concurrency_key, run_id, acquired_at, expires_at)
|
|
1041
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
1042
|
+
ON CONFLICT (workflow_name, concurrency_key, run_id) DO UPDATE SET
|
|
1043
|
+
expires_at = EXCLUDED.expires_at`, [workflowName, concurrencyKey, runId, Date.now(), leaseExpiresAt])
|
|
1044
|
+
.then(() => { }));
|
|
1045
|
+
}
|
|
1046
|
+
return result;
|
|
1047
|
+
}
|
|
1048
|
+
releaseConcurrencySlot(workflowName, concurrencyKey, runId) {
|
|
1049
|
+
this.memory.releaseConcurrencySlot(workflowName, concurrencyKey, runId);
|
|
1050
|
+
this.enqueueWrite(() => this.pool
|
|
1051
|
+
.query("DELETE FROM concurrency_locks WHERE workflow_name = $1 AND concurrency_key = $2 AND run_id = $3", [
|
|
1052
|
+
workflowName,
|
|
1053
|
+
concurrencyKey,
|
|
1054
|
+
runId,
|
|
1055
|
+
])
|
|
1056
|
+
.then(() => { }));
|
|
1057
|
+
}
|
|
1058
|
+
purgeExpiredConcurrencySlots(now) {
|
|
1059
|
+
const removed = this.memory.purgeExpiredConcurrencySlots(now);
|
|
1060
|
+
this.enqueueWrite(() => this.pool.query("DELETE FROM concurrency_locks WHERE expires_at <= $1", [now]).then(() => { }));
|
|
1061
|
+
return removed;
|
|
1062
|
+
}
|
|
1063
|
+
getConcurrencySnapshot(now) {
|
|
1064
|
+
return this.memory.getConcurrencySnapshot(now);
|
|
1065
|
+
}
|
|
1066
|
+
// === Durable scheduling (Tier 2 #5+#7 follow-up) ===
|
|
1067
|
+
// Tier 2 follow-up (migration v3) — PG mirror is now real durable
|
|
1068
|
+
// storage. Boot recovery (`HttpTrigger.recoverDispatches`) reads the
|
|
1069
|
+
// in-memory mirror, which is rehydrated from PG on init.
|
|
1070
|
+
upsertScheduledDispatch(row) {
|
|
1071
|
+
this.memory.upsertScheduledDispatch(row);
|
|
1072
|
+
// Tier C #2 — preserve claimed_by + claimed_at on conflict, same
|
|
1073
|
+
// invariant as sqlite. Re-upserts (debounce reset, queue re-defer)
|
|
1074
|
+
// MUST NOT release the claim or peers would re-claim the same row.
|
|
1075
|
+
this.enqueueWrite(() => this.pool
|
|
1076
|
+
.query(`INSERT INTO scheduled_dispatches
|
|
1077
|
+
(run_id, workflow_name, trigger_type, scheduled_at, expires_at,
|
|
1078
|
+
dispatch_status, payload_json, created_at)
|
|
1079
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
1080
|
+
ON CONFLICT (run_id) DO UPDATE SET
|
|
1081
|
+
scheduled_at = EXCLUDED.scheduled_at,
|
|
1082
|
+
expires_at = EXCLUDED.expires_at,
|
|
1083
|
+
dispatch_status = EXCLUDED.dispatch_status,
|
|
1084
|
+
payload_json = EXCLUDED.payload_json`, [
|
|
1085
|
+
row.runId,
|
|
1086
|
+
row.workflowName,
|
|
1087
|
+
row.triggerType,
|
|
1088
|
+
row.scheduledAt,
|
|
1089
|
+
row.expiresAt ?? null,
|
|
1090
|
+
row.dispatchStatus,
|
|
1091
|
+
JSON.stringify(row.payload ?? null),
|
|
1092
|
+
row.createdAt,
|
|
1093
|
+
])
|
|
1094
|
+
.then(() => { }));
|
|
1095
|
+
}
|
|
1096
|
+
// === Tier C #2 — cross-process scheduler coordination ===
|
|
1097
|
+
//
|
|
1098
|
+
// These methods bypass the in-memory mirror and hit PG directly —
|
|
1099
|
+
// claim atomicity REQUIRES the database to be the single source of
|
|
1100
|
+
// truth. The mirror is refreshed lazily as queries return.
|
|
1101
|
+
async claimDispatchesAsync(processId, leaseMs, now, opts) {
|
|
1102
|
+
const triggerClause = opts?.triggerType ? "AND trigger_type = $4" : "";
|
|
1103
|
+
const args = [processId, now, leaseMs];
|
|
1104
|
+
if (opts?.triggerType)
|
|
1105
|
+
args.push(opts.triggerType);
|
|
1106
|
+
const sql = `
|
|
1107
|
+
UPDATE scheduled_dispatches
|
|
1108
|
+
SET claimed_by = $1, claimed_at = $2
|
|
1109
|
+
WHERE (claimed_by IS NULL OR claimed_at + $3 < $2) ${triggerClause}
|
|
1110
|
+
RETURNING *
|
|
1111
|
+
`;
|
|
1112
|
+
const { rows } = await this.pool.query(sql, args);
|
|
1113
|
+
const out = rows.map((r) => ({
|
|
1114
|
+
runId: r.run_id,
|
|
1115
|
+
workflowName: r.workflow_name,
|
|
1116
|
+
triggerType: r.trigger_type,
|
|
1117
|
+
scheduledAt: Number(r.scheduled_at),
|
|
1118
|
+
expiresAt: r.expires_at !== null && r.expires_at !== undefined ? Number(r.expires_at) : undefined,
|
|
1119
|
+
dispatchStatus: r.dispatch_status,
|
|
1120
|
+
payload: typeof r.payload_json === "string" ? JSON.parse(r.payload_json) : r.payload_json,
|
|
1121
|
+
createdAt: Number(r.created_at),
|
|
1122
|
+
claimedBy: r.claimed_by !== null && r.claimed_by !== undefined ? r.claimed_by : undefined,
|
|
1123
|
+
claimedAt: r.claimed_at !== null && r.claimed_at !== undefined ? Number(r.claimed_at) : undefined,
|
|
1124
|
+
}));
|
|
1125
|
+
// Refresh the mirror so subsequent sync reads see the claim.
|
|
1126
|
+
for (const row of out)
|
|
1127
|
+
this.memory.upsertScheduledDispatch(row);
|
|
1128
|
+
out.sort((a, b) => a.scheduledAt - b.scheduledAt);
|
|
1129
|
+
return out;
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Sync wrapper required by the `RunStore` interface. PG's claim API
|
|
1133
|
+
* is fundamentally async (network round-trip); the sync wrapper
|
|
1134
|
+
* returns the LAST snapshot from the in-memory mirror.
|
|
1135
|
+
*
|
|
1136
|
+
* **For cross-process correctness, callers MUST use
|
|
1137
|
+
* `claimDispatchesAsync()` directly.** The sync wrapper exists so
|
|
1138
|
+
* the interface contract is satisfied for sqlite + in-memory
|
|
1139
|
+
* callers; PG users opt into the async path via the trigger's
|
|
1140
|
+
* recovery code (`HttpTrigger.recoverDispatches()` calls the
|
|
1141
|
+
* async version when the store is a `PostgresRunStore`).
|
|
1142
|
+
*/
|
|
1143
|
+
claimDispatches(processId, leaseMs, now, opts) {
|
|
1144
|
+
// Fire the async claim; return whatever the mirror has right now
|
|
1145
|
+
// (won't include rows claimed in this call until the async query
|
|
1146
|
+
// resolves and refreshes the mirror).
|
|
1147
|
+
void this.claimDispatchesAsync(processId, leaseMs, now, opts);
|
|
1148
|
+
return this.memory.claimDispatches(processId, leaseMs, now, opts);
|
|
1149
|
+
}
|
|
1150
|
+
heartbeatClaims(processId, now) {
|
|
1151
|
+
const count = this.memory.heartbeatClaims(processId, now);
|
|
1152
|
+
this.enqueueWrite(() => this.pool
|
|
1153
|
+
.query("UPDATE scheduled_dispatches SET claimed_at = $1 WHERE claimed_by = $2", [now, processId])
|
|
1154
|
+
.then(() => { }));
|
|
1155
|
+
return count;
|
|
1156
|
+
}
|
|
1157
|
+
releaseClaim(runId) {
|
|
1158
|
+
const removed = this.memory.releaseClaim(runId);
|
|
1159
|
+
this.enqueueWrite(() => this.pool
|
|
1160
|
+
.query("UPDATE scheduled_dispatches SET claimed_by = NULL, claimed_at = NULL WHERE run_id = $1 AND claimed_by IS NOT NULL", [runId])
|
|
1161
|
+
.then(() => { }));
|
|
1162
|
+
return removed;
|
|
1163
|
+
}
|
|
1164
|
+
deleteScheduledDispatch(runId) {
|
|
1165
|
+
const removed = this.memory.deleteScheduledDispatch(runId);
|
|
1166
|
+
this.enqueueWrite(() => this.pool.query("DELETE FROM scheduled_dispatches WHERE run_id = $1", [runId]).then(() => { }));
|
|
1167
|
+
return removed;
|
|
1168
|
+
}
|
|
1169
|
+
getScheduledDispatches(opts) {
|
|
1170
|
+
return this.memory.getScheduledDispatches(opts);
|
|
1171
|
+
}
|
|
1172
|
+
purgeExpiredScheduledDispatches(now) {
|
|
1173
|
+
const removed = this.memory.purgeExpiredScheduledDispatches(now);
|
|
1174
|
+
this.enqueueWrite(() => this.pool
|
|
1175
|
+
.query("DELETE FROM scheduled_dispatches WHERE expires_at IS NOT NULL AND expires_at < $1", [now])
|
|
1176
|
+
.then(() => { }));
|
|
1177
|
+
return removed;
|
|
1178
|
+
}
|
|
516
1179
|
// === Write Queue ===
|
|
517
1180
|
enqueueWrite(fn) {
|
|
518
1181
|
if (this.closed)
|
|
@@ -565,9 +1228,25 @@ export class PostgresRunStore {
|
|
|
565
1228
|
metadata: row.metadata_json ? parseJson(row.metadata_json) : undefined,
|
|
566
1229
|
nodeCount: Number(row.node_count),
|
|
567
1230
|
completedNodes: Number(row.completed_nodes),
|
|
1231
|
+
// PR 1-5 polish · pre-v5 PG rows have NULL on the columns added
|
|
1232
|
+
// by migration v5. Mirror sqlite's `rowToRun`: NULL `environment`
|
|
1233
|
+
// reads as "production" (the legacy default scope) so historical
|
|
1234
|
+
// data still surfaces under the EnvChip default. Every other new
|
|
1235
|
+
// column maps NULL → undefined.
|
|
1236
|
+
environment: (row.environment ?? "production"),
|
|
1237
|
+
replayOf: row.replay_of ?? undefined,
|
|
1238
|
+
parentRunId: row.parent_run_id ?? undefined,
|
|
1239
|
+
parentNodeRunId: row.parent_node_run_id ?? undefined,
|
|
1240
|
+
scheduledAt: row.scheduled_at != null ? Number(row.scheduled_at) : undefined,
|
|
1241
|
+
expiresAt: row.expires_at != null ? Number(row.expires_at) : undefined,
|
|
1242
|
+
debounceKey: row.debounce_key ?? undefined,
|
|
1243
|
+
debounceMode: (row.debounce_mode ?? undefined),
|
|
1244
|
+
pingCount: row.ping_count != null ? Number(row.ping_count) : undefined,
|
|
1245
|
+
lastCompletedStepIndex: row.last_completed_step_index != null ? Number(row.last_completed_step_index) : undefined,
|
|
568
1246
|
};
|
|
569
1247
|
}
|
|
570
1248
|
rowToNodeRun(row) {
|
|
1249
|
+
const flags = (row.flags_json ? parseJson(row.flags_json) : undefined);
|
|
571
1250
|
return {
|
|
572
1251
|
id: row.id,
|
|
573
1252
|
runId: row.run_id,
|
|
@@ -585,6 +1264,11 @@ export class PostgresRunStore {
|
|
|
585
1264
|
depth: Number(row.depth),
|
|
586
1265
|
stepIndex: Number(row.step_index),
|
|
587
1266
|
metrics: row.metrics_json ? parseJson(row.metrics_json) : undefined,
|
|
1267
|
+
wait: flags?.wait,
|
|
1268
|
+
dispatch: flags?.dispatch,
|
|
1269
|
+
subworkflowDepth: flags?.subworkflowDepth,
|
|
1270
|
+
middleware: flags?.middleware,
|
|
1271
|
+
iterationIndex: flags?.iterationIndex,
|
|
588
1272
|
};
|
|
589
1273
|
}
|
|
590
1274
|
rowToEvent(row) {
|
|
@@ -637,4 +1321,25 @@ function parseJson(value) {
|
|
|
637
1321
|
}
|
|
638
1322
|
return value;
|
|
639
1323
|
}
|
|
1324
|
+
/**
|
|
1325
|
+
* Serialize the in-memory NodeRun flag bag for the `flags_json` JSONB
|
|
1326
|
+
* column. Mirrors `encodeNodeRunFlags` in SqliteRunStore. Returns null
|
|
1327
|
+
* when no flags are set so the column stays NULL on the common case.
|
|
1328
|
+
*/
|
|
1329
|
+
function encodeNodeRunFlagsForPg(nodeRun) {
|
|
1330
|
+
const flags = {};
|
|
1331
|
+
if (nodeRun.wait !== undefined)
|
|
1332
|
+
flags.wait = nodeRun.wait;
|
|
1333
|
+
if (nodeRun.dispatch !== undefined)
|
|
1334
|
+
flags.dispatch = nodeRun.dispatch;
|
|
1335
|
+
if (nodeRun.subworkflowDepth !== undefined)
|
|
1336
|
+
flags.subworkflowDepth = nodeRun.subworkflowDepth;
|
|
1337
|
+
if (nodeRun.middleware !== undefined)
|
|
1338
|
+
flags.middleware = nodeRun.middleware;
|
|
1339
|
+
if (nodeRun.iterationIndex !== undefined)
|
|
1340
|
+
flags.iterationIndex = nodeRun.iterationIndex;
|
|
1341
|
+
if (Object.keys(flags).length === 0)
|
|
1342
|
+
return null;
|
|
1343
|
+
return JSON.stringify(flags);
|
|
1344
|
+
}
|
|
640
1345
|
//# sourceMappingURL=PostgresRunStore.js.map
|