@kubun/store-workflow 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/lib/api.d.ts +7 -1
  2. package/lib/api.js +147 -88
  3. package/lib/index.d.ts +1 -1
  4. package/lib/migrations.js +13 -0
  5. package/package.json +7 -7
package/lib/api.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { Adapter } from '@kubun/db-adapter';
2
- import type { Kysely } from 'kysely';
2
+ import { type Kysely } from 'kysely';
3
3
  import type { WorkflowInstance, WorkflowSchedule, WorkflowTables } from './tables.js';
4
4
  export type WorkflowStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
5
5
  export type WorkflowNextAction = {
@@ -46,6 +46,10 @@ export type FinishInput = {
46
46
  status: 'completed' | 'cancelled';
47
47
  state: unknown;
48
48
  };
49
+ export type PurgeFinishedInput = {
50
+ before: number;
51
+ limit: number;
52
+ };
49
53
  export type ListFilter = {
50
54
  name?: string;
51
55
  queue?: string;
@@ -85,6 +89,7 @@ export type UpsertAdaptiveScheduleInput = {
85
89
  };
86
90
  export type WorkflowStoreAPI = {
87
91
  insertInstance(input: InsertInstanceInput): Promise<InsertInstanceResult>;
92
+ insertInstanceTx(input: InsertInstanceInput): Promise<InsertInstanceResult>;
88
93
  claim(instanceID: string, bootID: string, leaseMs: number, now: number): Promise<WorkflowInstanceRow | null>;
89
94
  claimDue(queue: string, limit: number, bootID: string, leaseMs: number, now: number): Promise<Array<WorkflowInstanceRow>>;
90
95
  renew(instanceID: string, bootID: string, leaseMs: number, now: number): Promise<boolean>;
@@ -97,6 +102,7 @@ export type WorkflowStoreAPI = {
97
102
  listRunnableQueues(now: number): Promise<Array<string>>;
98
103
  get(instanceID: string): Promise<WorkflowInstanceRow | null>;
99
104
  list(filter?: ListFilter): Promise<Array<WorkflowInstanceRow>>;
105
+ purgeFinished(input: PurgeFinishedInput): Promise<number>;
100
106
  insertSchedule(input: InsertScheduleInput, now: number): Promise<void>;
101
107
  upsertAdaptiveSchedule(input: UpsertAdaptiveScheduleInput, now: number): Promise<WorkflowScheduleRow>;
102
108
  getSchedule(id: string): Promise<WorkflowScheduleRow | null>;
package/lib/api.js CHANGED
@@ -1,3 +1,11 @@
1
+ import { sql } from 'kysely';
2
+ // The statuses a run never leaves on its own. `failed` is one: only an
3
+ // explicit `retryFailed` re-arms it, and not once retention has deleted it.
4
+ const FINISHED_STATUSES = [
5
+ 'completed',
6
+ 'failed',
7
+ 'cancelled'
8
+ ];
1
9
  // --- Serialization helpers ---
2
10
  // ParseJSONResultsPlugin usually parses JSON columns back to objects, but a
3
11
  // value can still arrive as a string (adapter/driver dependent). Handle both.
@@ -30,100 +38,108 @@ export function createWorkflowStore(db, adapter) {
30
38
  async function readRow(tx, instanceID) {
31
39
  return await tx.selectFrom('kubun_workflow_instances').selectAll().where('instance_id', '=', instanceID).executeTakeFirst();
32
40
  }
41
+ // Shared body for insertInstance/insertInstanceTx: the pre-checks + guarded
42
+ // insert, run against whichever `tx` the caller hands in. insertInstance
43
+ // opens its own `db.transaction()` and passes that; insertInstanceTx passes
44
+ // `db` directly, assuming it is already the caller's open transaction.
45
+ async function insertInstanceOnTx(tx, input) {
46
+ const queue = input.queue ?? input.name;
47
+ const runAfter = input.runAfter ?? input.createdAt;
48
+ const updatedAt = input.updatedAt ?? input.createdAt;
49
+ if (input.idempotencyKey != null) {
50
+ const existing = await tx.selectFrom('kubun_workflow_instances').selectAll().where('idempotency_key', '=', input.idempotencyKey).executeTakeFirst();
51
+ if (existing != null) {
52
+ return {
53
+ row: rowToInstance(existing),
54
+ created: false,
55
+ reason: 'idempotent'
56
+ };
57
+ }
58
+ }
59
+ if (input.singletonKey != null) {
60
+ const active = await tx.selectFrom('kubun_workflow_instances').selectAll().where('name', '=', input.name).where('singleton_key', '=', input.singletonKey).where('status', 'in', [
61
+ 'pending',
62
+ 'running'
63
+ ]).executeTakeFirst();
64
+ if (active != null) {
65
+ return {
66
+ row: rowToInstance(active),
67
+ created: false,
68
+ reason: 'singleton'
69
+ };
70
+ }
71
+ }
72
+ // ON CONFLICT DO NOTHING makes the insert atomic against a racer this
73
+ // snapshot's pre-checks could not yet see: two inserts for the same
74
+ // idempotency_key (a retried remote enqueue) or the same active
75
+ // singleton both clear the pre-checks, and the unique indexes arbitrate.
76
+ // The loser inserts nothing and reads the winner's row below — no
77
+ // adapter-specific constraint-error parsing.
78
+ await tx.insertInto('kubun_workflow_instances').values({
79
+ instance_id: input.instanceID,
80
+ name: input.name,
81
+ queue,
82
+ idempotency_key: input.idempotencyKey ?? null,
83
+ singleton_key: input.singletonKey ?? null,
84
+ priority: input.priority ?? 0,
85
+ state: encodeJSON(input.state ?? {}),
86
+ next_action: encodeNextAction(input.nextAction ?? null),
87
+ status: 'pending',
88
+ attempts: 0,
89
+ run_after: runAfter,
90
+ lease_owner: null,
91
+ lease_expires: null,
92
+ last_error: null,
93
+ created_at: input.createdAt,
94
+ updated_at: updatedAt
95
+ }).onConflict((oc)=>oc.doNothing()).execute();
96
+ const row = await tx.selectFrom('kubun_workflow_instances').selectAll().where('instance_id', '=', input.instanceID).executeTakeFirst();
97
+ if (row != null) {
98
+ return {
99
+ row: rowToInstance(row),
100
+ created: true
101
+ };
102
+ }
103
+ // The insert was a no-op: a concurrent racer holds the conflicting key.
104
+ // Return its row as the idempotent/singleton hit, mirroring the pre-checks.
105
+ if (input.idempotencyKey != null) {
106
+ const existing = await tx.selectFrom('kubun_workflow_instances').selectAll().where('idempotency_key', '=', input.idempotencyKey).executeTakeFirst();
107
+ if (existing != null) {
108
+ return {
109
+ row: rowToInstance(existing),
110
+ created: false,
111
+ reason: 'idempotent'
112
+ };
113
+ }
114
+ }
115
+ if (input.singletonKey != null) {
116
+ const active = await tx.selectFrom('kubun_workflow_instances').selectAll().where('name', '=', input.name).where('singleton_key', '=', input.singletonKey).where('status', 'in', [
117
+ 'pending',
118
+ 'running'
119
+ ]).executeTakeFirst();
120
+ if (active != null) {
121
+ return {
122
+ row: rowToInstance(active),
123
+ created: false,
124
+ reason: 'singleton'
125
+ };
126
+ }
127
+ }
128
+ // A no-op insert with no recoverable conflicting row means the constraint
129
+ // hit was neither unique index we recover from — surface it, never a
130
+ // silent empty return.
131
+ throw new Error(`Workflow instance "${input.instanceID}" insert did nothing but no conflicting row was found`);
132
+ }
33
133
  return {
34
134
  async insertInstance (input) {
35
- const queue = input.queue ?? input.name;
36
- const runAfter = input.runAfter ?? input.createdAt;
37
- const updatedAt = input.updatedAt ?? input.createdAt;
38
135
  // The transaction gives the pre-checks and the insert one snapshot. The
39
136
  // partial unique index is the race backstop; the graceful created:false
40
137
  // return is the sequential pre-check path, so no driver error strings are
41
138
  // parsed.
42
- return await db.transaction().execute(async (tx)=>{
43
- if (input.idempotencyKey != null) {
44
- const existing = await tx.selectFrom('kubun_workflow_instances').selectAll().where('idempotency_key', '=', input.idempotencyKey).executeTakeFirst();
45
- if (existing != null) {
46
- return {
47
- row: rowToInstance(existing),
48
- created: false,
49
- reason: 'idempotent'
50
- };
51
- }
52
- }
53
- if (input.singletonKey != null) {
54
- const active = await tx.selectFrom('kubun_workflow_instances').selectAll().where('name', '=', input.name).where('singleton_key', '=', input.singletonKey).where('status', 'in', [
55
- 'pending',
56
- 'running'
57
- ]).executeTakeFirst();
58
- if (active != null) {
59
- return {
60
- row: rowToInstance(active),
61
- created: false,
62
- reason: 'singleton'
63
- };
64
- }
65
- }
66
- // ON CONFLICT DO NOTHING makes the insert atomic against a racer this
67
- // snapshot's pre-checks could not yet see: two inserts for the same
68
- // idempotency_key (the cross-hub idempotent retry) or the same active
69
- // singleton both clear the pre-checks, and the unique indexes arbitrate.
70
- // The loser inserts nothing and reads the winner's row below — no
71
- // adapter-specific constraint-error parsing.
72
- await tx.insertInto('kubun_workflow_instances').values({
73
- instance_id: input.instanceID,
74
- name: input.name,
75
- queue,
76
- idempotency_key: input.idempotencyKey ?? null,
77
- singleton_key: input.singletonKey ?? null,
78
- priority: input.priority ?? 0,
79
- state: encodeJSON(input.state ?? {}),
80
- next_action: encodeNextAction(input.nextAction ?? null),
81
- status: 'pending',
82
- attempts: 0,
83
- run_after: runAfter,
84
- lease_owner: null,
85
- lease_expires: null,
86
- last_error: null,
87
- created_at: input.createdAt,
88
- updated_at: updatedAt
89
- }).onConflict((oc)=>oc.doNothing()).execute();
90
- const row = await tx.selectFrom('kubun_workflow_instances').selectAll().where('instance_id', '=', input.instanceID).executeTakeFirst();
91
- if (row != null) {
92
- return {
93
- row: rowToInstance(row),
94
- created: true
95
- };
96
- }
97
- // The insert was a no-op: a concurrent racer holds the conflicting key.
98
- // Return its row as the idempotent/singleton hit, mirroring the pre-checks.
99
- if (input.idempotencyKey != null) {
100
- const existing = await tx.selectFrom('kubun_workflow_instances').selectAll().where('idempotency_key', '=', input.idempotencyKey).executeTakeFirst();
101
- if (existing != null) {
102
- return {
103
- row: rowToInstance(existing),
104
- created: false,
105
- reason: 'idempotent'
106
- };
107
- }
108
- }
109
- if (input.singletonKey != null) {
110
- const active = await tx.selectFrom('kubun_workflow_instances').selectAll().where('name', '=', input.name).where('singleton_key', '=', input.singletonKey).where('status', 'in', [
111
- 'pending',
112
- 'running'
113
- ]).executeTakeFirst();
114
- if (active != null) {
115
- return {
116
- row: rowToInstance(active),
117
- created: false,
118
- reason: 'singleton'
119
- };
120
- }
121
- }
122
- // A no-op insert with no recoverable conflicting row means the constraint
123
- // hit was neither unique index we recover from — surface it, never a
124
- // silent empty return.
125
- throw new Error(`Workflow instance "${input.instanceID}" insert did nothing but no conflicting row was found`);
126
- });
139
+ return await db.transaction().execute((tx)=>insertInstanceOnTx(tx, input));
140
+ },
141
+ async insertInstanceTx (input) {
142
+ return await insertInstanceOnTx(db, input);
127
143
  },
128
144
  async claim (instanceID, bootID, leaseMs, now) {
129
145
  // One atomic UPDATE. Claimable when pending, or when a running lease has
@@ -273,6 +289,49 @@ export function createWorkflowStore(db, adapter) {
273
289
  const rows = await query.execute();
274
290
  return rows.map(rowToInstance);
275
291
  },
292
+ async purgeFinished (input) {
293
+ if (!Number.isFinite(input.before)) {
294
+ throw new Error('purgeFinished: before must be a finite number');
295
+ }
296
+ if (!Number.isInteger(input.limit) || input.limit <= 0) {
297
+ throw new Error('purgeFinished: limit must be a positive integer');
298
+ }
299
+ const { before, limit } = input;
300
+ // Finished before the cutoff, and holding no lease that outlives it.
301
+ const finished = (eb)=>eb.and([
302
+ eb('status', 'in', FINISHED_STATUSES),
303
+ eb('updated_at', '<', before),
304
+ eb.or([
305
+ eb('lease_expires', 'is', null),
306
+ eb('lease_expires', '<', before)
307
+ ])
308
+ ]);
309
+ // One statement, so the keep rule and the delete see the same rows. A
310
+ // row of a scheduled subject is only deleted while a newer row of the
311
+ // same (name, singleton_key) exists; the newest never has one, so no
312
+ // batch and no concurrent cleanup can delete it. A row only gets newer
313
+ // by being written, and a written row is no longer past the cutoff.
314
+ const result = await db.deleteFrom('kubun_workflow_instances').where('instance_id', 'in', (eb)=>eb.selectFrom('kubun_workflow_instances as i').select('i.instance_id').where('i.status', 'in', FINISHED_STATUSES).where('i.updated_at', '<', before).where((inner)=>inner.or([
315
+ inner('i.lease_expires', 'is', null),
316
+ inner('i.lease_expires', '<', before)
317
+ ])).where((inner)=>inner.or([
318
+ inner('i.singleton_key', 'is', null),
319
+ inner.not(inner.exists(inner.selectFrom('kubun_workflow_schedules as s').select('s.id').whereRef('s.workflow_name', '=', 'i.name')// Adaptive schedule ids are `<name>:<subject>`; the
320
+ // `||` concatenation is portable across SQLite and Postgres.
321
+ .where('s.id', '=', sql`${sql.ref('i.name')} || ':' || ${sql.ref('i.singleton_key')}`))),
322
+ inner.exists(inner.selectFrom('kubun_workflow_instances as n').select('n.instance_id').whereRef('n.name', '=', 'i.name').whereRef('n.singleton_key', '=', 'i.singleton_key').where((order)=>order.or([
323
+ order('n.updated_at', '>', order.ref('i.updated_at')),
324
+ order.and([
325
+ order('n.updated_at', '=', order.ref('i.updated_at')),
326
+ order('n.instance_id', '>', order.ref('i.instance_id'))
327
+ ])
328
+ ])))
329
+ ])).orderBy('i.updated_at').orderBy('i.instance_id').limit(limit))// Re-checked on the row being deleted: a row re-armed or claimed
330
+ // between the candidate read and the delete (Postgres, read committed)
331
+ // is kept.
332
+ .where(finished).executeTakeFirst();
333
+ return Number(result?.numDeletedRows ?? 0n);
334
+ },
276
335
  // --- Schedules ---
277
336
  async insertSchedule (input, now) {
278
337
  await db.insertInto('kubun_workflow_schedules').values({
package/lib/index.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import type { StoreProvider } from '@kubun/db';
2
2
  import type { WorkflowStoreAPI } from './api.js';
3
- export type { AdvanceScheduleAdaptiveInput, CheckpointInput, FailInput, FinishInput, InsertInstanceInput, InsertInstanceResult, InsertReason, InsertScheduleInput, ListFilter, WorkflowInstanceRow, WorkflowNextAction, WorkflowScheduleRow, WorkflowStatus, WorkflowStoreAPI, } from './api.js';
3
+ export type { AdvanceScheduleAdaptiveInput, CheckpointInput, FailInput, FinishInput, InsertInstanceInput, InsertInstanceResult, InsertReason, InsertScheduleInput, ListFilter, PurgeFinishedInput, WorkflowInstanceRow, WorkflowNextAction, WorkflowScheduleRow, WorkflowStatus, WorkflowStoreAPI, } from './api.js';
4
4
  export { createWorkflowStore } from './api.js';
5
5
  export { workflowStoreDefinition } from './definition.js';
6
6
  export { WORKFLOW_SINGLETON_UNIQUE_INDEX } from './migrations.js';
package/lib/migrations.js CHANGED
@@ -38,6 +38,17 @@ export function getWorkflowMigrations(ctx) {
38
38
  'status',
39
39
  'lease_expires'
40
40
  ]).execute();
41
+ // Retention scan: finished rows oldest first.
42
+ await db.schema.createIndex('idx_workflow_instances_finished').ifNotExists().on('kubun_workflow_instances').columns([
43
+ 'status',
44
+ 'updated_at'
45
+ ]).execute();
46
+ // Newest instance of a singleton subject: the row retention keeps.
47
+ await db.schema.createIndex('idx_workflow_instances_subject').ifNotExists().on('kubun_workflow_instances').columns([
48
+ 'name',
49
+ 'singleton_key',
50
+ 'updated_at'
51
+ ]).execute();
41
52
  await db.schema.createTable('kubun_workflow_schedules').ifNotExists().addColumn('id', t.text, (col)=>col.notNull().primaryKey()).addColumn('workflow_name', t.text, (col)=>col.notNull()).addColumn('params', t.json, (col)=>col.notNull())// Exactly one of cron / every is set; both nullable, the store persists
42
53
  // them without parsing.
43
54
  .addColumn('cron', t.text).addColumn('every', t.text)// Default keeps existing interval inserts working unchanged.
@@ -54,6 +65,8 @@ export function getWorkflowMigrations(ctx) {
54
65
  async down (db) {
55
66
  await db.schema.dropIndex('idx_workflow_schedules_due').ifExists().execute();
56
67
  await db.schema.dropTable('kubun_workflow_schedules').ifExists().execute();
68
+ await db.schema.dropIndex('idx_workflow_instances_subject').ifExists().execute();
69
+ await db.schema.dropIndex('idx_workflow_instances_finished').ifExists().execute();
57
70
  await db.schema.dropIndex('idx_workflow_instances_recovery').ifExists().execute();
58
71
  await db.schema.dropIndex('idx_workflow_instances_claim').ifExists().execute();
59
72
  await db.schema.dropIndex(WORKFLOW_SINGLETON_UNIQUE_INDEX).ifExists().execute();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kubun/store-workflow",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "keywords": [],
5
5
  "license": "see LICENSE.md",
6
6
  "sideEffects": false,
@@ -15,14 +15,14 @@
15
15
  "LICENSE.md"
16
16
  ],
17
17
  "dependencies": {
18
- "@kubun/db": "^0.17.0",
19
- "@kubun/db-adapter": "^0.17.0",
20
- "kysely": "^0.29.5"
18
+ "@kubun/db": "^0.18.0",
19
+ "@kubun/db-adapter": "^0.18.0",
20
+ "kysely": "^0.29.7"
21
21
  },
22
22
  "devDependencies": {
23
- "@kubun/db-better-sqlite": "^0.17.0",
24
- "@kubun/db-postgres": "^0.17.0",
25
- "@testcontainers/postgresql": "^12.1.0"
23
+ "@kubun/db-better-sqlite": "^0.18.0",
24
+ "@kubun/db-postgres": "^0.18.0",
25
+ "@testcontainers/postgresql": "^12.2.1"
26
26
  },
27
27
  "publishConfig": {
28
28
  "access": "public"