@pikku/kysely 0.13.4 → 0.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -0
- package/LICENSE +21 -0
- package/README.md +29 -2
- package/dist/src/kysely-tables.d.ts +8 -0
- package/dist/src/kysely-workflow-mirror.js +37 -25
- package/dist/src/kysely-workflow-run-service.d.ts +0 -5
- package/dist/src/kysely-workflow-run-service.js +4 -16
- package/dist/src/kysely-workflow-service.d.ts +35 -6
- package/dist/src/kysely-workflow-service.js +194 -134
- package/dist/src/schema/index.d.ts +17 -0
- package/dist/src/schema/index.js +96 -9
- package/dist/src/schema/workflow.schema.d.ts +4 -2
- package/dist/src/schema/workflow.schema.js +28 -2
- package/package.json +3 -2
- package/src/kysely-tables.ts +8 -0
- package/src/kysely-workflow-mirror.test.ts +1 -13
- package/src/kysely-workflow-mirror.ts +44 -20
- package/src/kysely-workflow-run-service.ts +7 -20
- package/src/kysely-workflow-service.test.ts +431 -0
- package/src/kysely-workflow-service.ts +212 -161
- package/src/schema/index.ts +138 -13
- package/src/schema/schema.test.ts +183 -0
- package/src/schema/workflow.schema.ts +38 -2
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { PikkuWorkflowService, } from '@pikku/core/workflow';
|
|
2
|
+
import { sql } from 'kysely';
|
|
2
3
|
import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js';
|
|
3
4
|
import { parseJson } from './kysely-json.js';
|
|
4
5
|
import { ensurePikkuSchema } from './schema/index.js';
|
|
@@ -68,11 +69,12 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
68
69
|
retries: stepOptions?.retries ?? null,
|
|
69
70
|
retryDelay: stepOptions?.retryDelay?.toString() ?? null,
|
|
70
71
|
fromStepName: fromStepName ?? null,
|
|
72
|
+
currentAttempt: 1,
|
|
71
73
|
createdAt: now,
|
|
72
74
|
updatedAt: now,
|
|
73
75
|
})
|
|
74
76
|
.execute();
|
|
75
|
-
await this.insertHistoryRecord(stepId, 'pending');
|
|
77
|
+
await this.insertHistoryRecord(stepId, 'pending', 1);
|
|
76
78
|
return {
|
|
77
79
|
stepId,
|
|
78
80
|
status: 'pending',
|
|
@@ -88,25 +90,24 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
88
90
|
}
|
|
89
91
|
async getStepState(runId, stepName) {
|
|
90
92
|
const row = await this.db
|
|
91
|
-
.selectFrom('workflowStep
|
|
93
|
+
.selectFrom('workflowStep')
|
|
92
94
|
.select([
|
|
93
|
-
'
|
|
94
|
-
'
|
|
95
|
-
'
|
|
96
|
-
'
|
|
97
|
-
'
|
|
98
|
-
'
|
|
99
|
-
'
|
|
100
|
-
|
|
101
|
-
|
|
95
|
+
'workflowStepId',
|
|
96
|
+
'status',
|
|
97
|
+
'result',
|
|
98
|
+
'error',
|
|
99
|
+
'retries',
|
|
100
|
+
'retryDelay',
|
|
101
|
+
'fromStepName',
|
|
102
|
+
// `current_attempt` is bumped alongside every history insert, so it is
|
|
103
|
+
// already the count this used to aggregate — and reading it keeps the
|
|
104
|
+
// engine's hottest query off a table that grows for the life of the run.
|
|
105
|
+
'currentAttempt',
|
|
106
|
+
'createdAt',
|
|
107
|
+
'updatedAt',
|
|
102
108
|
])
|
|
103
|
-
.
|
|
104
|
-
.
|
|
105
|
-
.select(eb.fn.countAll().as('cnt'))
|
|
106
|
-
.whereRef('workflowStepHistory.workflowStepId', '=', 's.workflowStepId')
|
|
107
|
-
.as('attemptCount'))
|
|
108
|
-
.where('s.workflowRunId', '=', runId)
|
|
109
|
-
.where('s.stepName', '=', stepName)
|
|
109
|
+
.where('workflowRunId', '=', runId)
|
|
110
|
+
.where('stepName', '=', stepName)
|
|
110
111
|
.executeTakeFirst();
|
|
111
112
|
if (!row) {
|
|
112
113
|
throw new Error(`Step not found: runId=${runId}, stepName=${stepName}. Use insertStepState to create it.`);
|
|
@@ -116,7 +117,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
116
117
|
status: row.status,
|
|
117
118
|
result: parseJson(row.result),
|
|
118
119
|
error: parseJson(row.error),
|
|
119
|
-
attemptCount: Number(row.
|
|
120
|
+
attemptCount: Number(row.currentAttempt ?? 1),
|
|
120
121
|
retries: row.retries != null ? Number(row.retries) : undefined,
|
|
121
122
|
retryDelay: row.retryDelay ?? undefined,
|
|
122
123
|
fromStepName: row.fromStepName ?? undefined,
|
|
@@ -124,29 +125,45 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
124
125
|
updatedAt: new Date(row.updatedAt),
|
|
125
126
|
};
|
|
126
127
|
}
|
|
128
|
+
async listStepStates(runId) {
|
|
129
|
+
const rows = await this.db
|
|
130
|
+
.selectFrom('workflowStep')
|
|
131
|
+
.select([
|
|
132
|
+
'workflowStepId',
|
|
133
|
+
'stepName',
|
|
134
|
+
'status',
|
|
135
|
+
'result',
|
|
136
|
+
'error',
|
|
137
|
+
'retries',
|
|
138
|
+
'retryDelay',
|
|
139
|
+
'fromStepName',
|
|
140
|
+
'currentAttempt',
|
|
141
|
+
'createdAt',
|
|
142
|
+
'updatedAt',
|
|
143
|
+
])
|
|
144
|
+
.where('workflowRunId', '=', runId)
|
|
145
|
+
.execute();
|
|
146
|
+
return rows.map((row) => ({
|
|
147
|
+
stepId: row.workflowStepId,
|
|
148
|
+
stepName: row.stepName,
|
|
149
|
+
status: row.status,
|
|
150
|
+
result: parseJson(row.result),
|
|
151
|
+
error: parseJson(row.error),
|
|
152
|
+
attemptCount: Number(row.currentAttempt ?? 1),
|
|
153
|
+
retries: row.retries != null ? Number(row.retries) : undefined,
|
|
154
|
+
retryDelay: row.retryDelay ?? undefined,
|
|
155
|
+
fromStepName: row.fromStepName ?? undefined,
|
|
156
|
+
createdAt: new Date(row.createdAt),
|
|
157
|
+
updatedAt: new Date(row.updatedAt),
|
|
158
|
+
}));
|
|
159
|
+
}
|
|
127
160
|
async getRunHistory(runId) {
|
|
128
161
|
return this.runService.getRunHistory(runId);
|
|
129
162
|
}
|
|
130
163
|
async setStepRunningImpl(stepId) {
|
|
131
|
-
await this.
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
.where('workflowStepId', '=', stepId)
|
|
135
|
-
.execute();
|
|
136
|
-
const latestHistory = await this.db
|
|
137
|
-
.selectFrom('workflowStepHistory')
|
|
138
|
-
.select('historyId')
|
|
139
|
-
.where('workflowStepId', '=', stepId)
|
|
140
|
-
.orderBy('createdAt', 'desc')
|
|
141
|
-
.limit(1)
|
|
142
|
-
.executeTakeFirst();
|
|
143
|
-
if (latestHistory) {
|
|
144
|
-
await this.db
|
|
145
|
-
.updateTable('workflowStepHistory')
|
|
146
|
-
.set({ status: 'running', runningAt: new Date() })
|
|
147
|
-
.where('historyId', '=', latestHistory.historyId)
|
|
148
|
-
.execute();
|
|
149
|
-
}
|
|
164
|
+
await this.writeStepTransition(stepId, 'running', {
|
|
165
|
+
runningAt: new Date(),
|
|
166
|
+
});
|
|
150
167
|
}
|
|
151
168
|
async setStepScheduledImpl(stepId) {
|
|
152
169
|
await this.db
|
|
@@ -155,12 +172,85 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
155
172
|
.where('workflowStepId', '=', stepId)
|
|
156
173
|
.execute();
|
|
157
174
|
}
|
|
158
|
-
|
|
175
|
+
/**
|
|
176
|
+
* Move a step and its current history attempt to the same status in one
|
|
177
|
+
* transaction.
|
|
178
|
+
*
|
|
179
|
+
* Both halves matter: a crash between them used to leave the step row saying
|
|
180
|
+
* `succeeded` while its history row still said `running`, which silently
|
|
181
|
+
* corrupts `getRunHistory` and every timeline reconstruction built on it.
|
|
182
|
+
* The history row is targeted by its `attempt` rather than by a `LIMIT 1`
|
|
183
|
+
* over `created_at`, so a retry that lands in the same millisecond as the
|
|
184
|
+
* attempt it replaces still resolves the newer row.
|
|
185
|
+
*/
|
|
186
|
+
async writeStepTransition(stepId, status, historyValues) {
|
|
187
|
+
const now = new Date();
|
|
188
|
+
const stepValues = { status, updatedAt: now };
|
|
189
|
+
if ('result' in historyValues)
|
|
190
|
+
stepValues.result = historyValues.result;
|
|
191
|
+
if ('error' in historyValues)
|
|
192
|
+
stepValues.error = historyValues.error;
|
|
193
|
+
// A step carries only its latest outcome, so the other half is cleared.
|
|
194
|
+
if (status === 'succeeded')
|
|
195
|
+
stepValues.error = null;
|
|
196
|
+
if (status === 'failed')
|
|
197
|
+
stepValues.result = null;
|
|
198
|
+
await this.db.transaction().execute(async (trx) => {
|
|
199
|
+
await trx
|
|
200
|
+
.updateTable('workflowStep')
|
|
201
|
+
.set({
|
|
202
|
+
...stepValues,
|
|
203
|
+
// A step that somehow reaches a transition without a live attempt
|
|
204
|
+
// addresses no history row at all, so give it one here. Written as an
|
|
205
|
+
// expression over the row's own column rather than a subquery, which
|
|
206
|
+
// MySQL would reject against the table being updated.
|
|
207
|
+
currentAttempt: sql `coalesce(current_attempt, 1)`,
|
|
208
|
+
})
|
|
209
|
+
.where('workflowStepId', '=', stepId)
|
|
210
|
+
.execute();
|
|
211
|
+
const written = await trx
|
|
212
|
+
.updateTable('workflowStepHistory')
|
|
213
|
+
.set({ status, ...historyValues })
|
|
214
|
+
.where('workflowStepId', '=', stepId)
|
|
215
|
+
// Correlate through the step row rather than a MAX() over this table:
|
|
216
|
+
// MySQL rejects a subquery that reads the table being updated, and a
|
|
217
|
+
// primary-key lookup is cheaper than an aggregate anyway.
|
|
218
|
+
.where('attempt', '=', (eb) => eb
|
|
219
|
+
.selectFrom('workflowStep')
|
|
220
|
+
.select('currentAttempt')
|
|
221
|
+
.where('workflowStepId', '=', stepId))
|
|
222
|
+
.executeTakeFirst();
|
|
223
|
+
// Matching nothing means the attempt has no history row to move, which
|
|
224
|
+
// would commit the step's new status with no record of it — precisely
|
|
225
|
+
// the divergence this transaction exists to prevent. Write the missing
|
|
226
|
+
// row instead of letting the pair drift apart.
|
|
227
|
+
if (Number(written?.numUpdatedRows ?? 0n) === 0) {
|
|
228
|
+
const step = await trx
|
|
229
|
+
.selectFrom('workflowStep')
|
|
230
|
+
.select('currentAttempt')
|
|
231
|
+
.where('workflowStepId', '=', stepId)
|
|
232
|
+
.executeTakeFirst();
|
|
233
|
+
await trx
|
|
234
|
+
.insertInto('workflowStepHistory')
|
|
235
|
+
.values({
|
|
236
|
+
historyId: crypto.randomUUID(),
|
|
237
|
+
workflowStepId: stepId,
|
|
238
|
+
status,
|
|
239
|
+
attempt: step?.currentAttempt ?? 1,
|
|
240
|
+
createdAt: now,
|
|
241
|
+
...historyValues,
|
|
242
|
+
})
|
|
243
|
+
.execute();
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
async insertHistoryRecord(stepId, status, attempt, result, error) {
|
|
159
248
|
const now = new Date();
|
|
160
249
|
const values = {
|
|
161
250
|
historyId: crypto.randomUUID(),
|
|
162
251
|
workflowStepId: stepId,
|
|
163
252
|
status,
|
|
253
|
+
attempt,
|
|
164
254
|
result: result != null ? JSON.stringify(result) : null,
|
|
165
255
|
error: error != null ? JSON.stringify(error) : null,
|
|
166
256
|
createdAt: now,
|
|
@@ -199,35 +289,10 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
199
289
|
.execute();
|
|
200
290
|
}
|
|
201
291
|
async setStepResultImpl(stepId, result) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
status: 'succeeded',
|
|
207
|
-
result: resultJson,
|
|
208
|
-
error: null,
|
|
209
|
-
updatedAt: new Date(),
|
|
210
|
-
})
|
|
211
|
-
.where('workflowStepId', '=', stepId)
|
|
212
|
-
.execute();
|
|
213
|
-
const latestHistory = await this.db
|
|
214
|
-
.selectFrom('workflowStepHistory')
|
|
215
|
-
.select('historyId')
|
|
216
|
-
.where('workflowStepId', '=', stepId)
|
|
217
|
-
.orderBy('createdAt', 'desc')
|
|
218
|
-
.limit(1)
|
|
219
|
-
.executeTakeFirst();
|
|
220
|
-
if (latestHistory) {
|
|
221
|
-
await this.db
|
|
222
|
-
.updateTable('workflowStepHistory')
|
|
223
|
-
.set({
|
|
224
|
-
status: 'succeeded',
|
|
225
|
-
result: resultJson,
|
|
226
|
-
succeededAt: new Date(),
|
|
227
|
-
})
|
|
228
|
-
.where('historyId', '=', latestHistory.historyId)
|
|
229
|
-
.execute();
|
|
230
|
-
}
|
|
292
|
+
await this.writeStepTransition(stepId, 'succeeded', {
|
|
293
|
+
result: JSON.stringify(result),
|
|
294
|
+
succeededAt: new Date(),
|
|
295
|
+
});
|
|
231
296
|
}
|
|
232
297
|
async setStepErrorImpl(stepId, error) {
|
|
233
298
|
const serializedError = {
|
|
@@ -235,65 +300,54 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
235
300
|
stack: error.stack,
|
|
236
301
|
code: error.code,
|
|
237
302
|
};
|
|
238
|
-
|
|
303
|
+
await this.writeStepTransition(stepId, 'failed', {
|
|
304
|
+
error: JSON.stringify(serializedError),
|
|
305
|
+
failedAt: new Date(),
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
async createRetryAttemptImpl(stepId, status) {
|
|
309
|
+
// Read before writing: the new attempt number comes from the history this
|
|
310
|
+
// step already has, and the row it reads is the one returned below.
|
|
311
|
+
const previous = await this.db
|
|
312
|
+
.selectFrom('workflowStep')
|
|
313
|
+
.select('currentAttempt')
|
|
314
|
+
.where('workflowStepId', '=', stepId)
|
|
315
|
+
.executeTakeFirst();
|
|
316
|
+
const attempt = (previous?.currentAttempt ?? 0) + 1;
|
|
239
317
|
await this.db
|
|
240
318
|
.updateTable('workflowStep')
|
|
241
319
|
.set({
|
|
242
|
-
status
|
|
243
|
-
error: errorJson,
|
|
320
|
+
status,
|
|
244
321
|
result: null,
|
|
322
|
+
error: null,
|
|
323
|
+
currentAttempt: attempt,
|
|
245
324
|
updatedAt: new Date(),
|
|
246
325
|
})
|
|
247
326
|
.where('workflowStepId', '=', stepId)
|
|
248
327
|
.execute();
|
|
249
|
-
|
|
250
|
-
.selectFrom('workflowStepHistory')
|
|
251
|
-
.select('historyId')
|
|
252
|
-
.where('workflowStepId', '=', stepId)
|
|
253
|
-
.orderBy('createdAt', 'desc')
|
|
254
|
-
.limit(1)
|
|
255
|
-
.executeTakeFirst();
|
|
256
|
-
if (latestHistory) {
|
|
257
|
-
await this.db
|
|
258
|
-
.updateTable('workflowStepHistory')
|
|
259
|
-
.set({ status: 'failed', error: errorJson, failedAt: new Date() })
|
|
260
|
-
.where('historyId', '=', latestHistory.historyId)
|
|
261
|
-
.execute();
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
async createRetryAttemptImpl(stepId, status) {
|
|
265
|
-
await this.db
|
|
266
|
-
.updateTable('workflowStep')
|
|
267
|
-
.set({ status, result: null, error: null, updatedAt: new Date() })
|
|
268
|
-
.where('workflowStepId', '=', stepId)
|
|
269
|
-
.execute();
|
|
270
|
-
await this.insertHistoryRecord(stepId, status);
|
|
328
|
+
await this.insertHistoryRecord(stepId, status, attempt);
|
|
271
329
|
const row = await this.db
|
|
272
|
-
.selectFrom('workflowStep
|
|
330
|
+
.selectFrom('workflowStep')
|
|
273
331
|
.select([
|
|
274
|
-
'
|
|
275
|
-
'
|
|
276
|
-
'
|
|
277
|
-
'
|
|
278
|
-
'
|
|
279
|
-
'
|
|
280
|
-
'
|
|
281
|
-
'
|
|
282
|
-
'
|
|
332
|
+
'workflowStepId',
|
|
333
|
+
'status',
|
|
334
|
+
'result',
|
|
335
|
+
'error',
|
|
336
|
+
'retries',
|
|
337
|
+
'retryDelay',
|
|
338
|
+
'fromStepName',
|
|
339
|
+
'currentAttempt',
|
|
340
|
+
'createdAt',
|
|
341
|
+
'updatedAt',
|
|
283
342
|
])
|
|
284
|
-
.
|
|
285
|
-
.selectFrom('workflowStepHistory')
|
|
286
|
-
.select(eb.fn.countAll().as('cnt'))
|
|
287
|
-
.whereRef('workflowStepHistory.workflowStepId', '=', 's.workflowStepId')
|
|
288
|
-
.as('attemptCount'))
|
|
289
|
-
.where('s.workflowStepId', '=', stepId)
|
|
343
|
+
.where('workflowStepId', '=', stepId)
|
|
290
344
|
.executeTakeFirstOrThrow();
|
|
291
345
|
return {
|
|
292
346
|
stepId: row.workflowStepId,
|
|
293
347
|
status: row.status,
|
|
294
348
|
result: parseJson(row.result),
|
|
295
349
|
error: parseJson(row.error),
|
|
296
|
-
attemptCount: Number(row.
|
|
350
|
+
attemptCount: Number(row.currentAttempt ?? 1),
|
|
297
351
|
retries: row.retries != null ? Number(row.retries) : undefined,
|
|
298
352
|
retryDelay: row.retryDelay ?? undefined,
|
|
299
353
|
fromStepName: row.fromStepName ?? undefined,
|
|
@@ -386,17 +440,39 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
386
440
|
.where('workflowStepId', '=', stepId)
|
|
387
441
|
.execute();
|
|
388
442
|
}
|
|
443
|
+
/**
|
|
444
|
+
* Merge one key into the run's JSON state, as a single expression evaluated
|
|
445
|
+
* by the database.
|
|
446
|
+
*
|
|
447
|
+
* The SQL is dialect-specific, so subclasses override this rather than the
|
|
448
|
+
* caller. The default is the SQLite form; MySQL and Postgres differ in how
|
|
449
|
+
* they cast a JSON literal.
|
|
450
|
+
*
|
|
451
|
+
* @param path - JSON path to the key, already quoted (`$."key"`)
|
|
452
|
+
* @param json - The value as JSON text
|
|
453
|
+
*/
|
|
454
|
+
jsonSetState(path, json) {
|
|
455
|
+
return sql `json_set(coalesce(state, '{}'), ${path}, json(${json}))`;
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* A JSON path for an arbitrary key. State keys are user-supplied (a graph's
|
|
459
|
+
* `setState` name, an approval's hex-encoded reason), so the key is quoted
|
|
460
|
+
* rather than interpolated bare — an unquoted `$.a.b` would silently address
|
|
461
|
+
* a nested path instead of the key literally called `a.b`.
|
|
462
|
+
*/
|
|
463
|
+
jsonPathFor(name) {
|
|
464
|
+
return `$.${JSON.stringify(name)}`;
|
|
465
|
+
}
|
|
389
466
|
async updateRunStateImpl(runId, name, value) {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
.where('workflowRunId', '=', runId)
|
|
394
|
-
.executeTakeFirst();
|
|
395
|
-
const state = parseJson(row?.state) ?? {};
|
|
396
|
-
state[name] = value;
|
|
467
|
+
// Merged by the database, not read-modify-written in JS: two graph nodes
|
|
468
|
+
// calling setState concurrently used to race, and the later write silently
|
|
469
|
+
// dropped whichever key the earlier one had added.
|
|
397
470
|
await this.db
|
|
398
471
|
.updateTable('workflowRuns')
|
|
399
|
-
.set({
|
|
472
|
+
.set({
|
|
473
|
+
state: this.jsonSetState(this.jsonPathFor(name), JSON.stringify(value ?? null)),
|
|
474
|
+
updatedAt: new Date(),
|
|
475
|
+
})
|
|
400
476
|
.where('workflowRunId', '=', runId)
|
|
401
477
|
.execute();
|
|
402
478
|
}
|
|
@@ -434,21 +510,5 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
434
510
|
async getWorkflowVersion(name, graphHash) {
|
|
435
511
|
return this.runService.getWorkflowVersion(name, graphHash);
|
|
436
512
|
}
|
|
437
|
-
async getAIGeneratedWorkflows(agentName) {
|
|
438
|
-
let query = this.db
|
|
439
|
-
.selectFrom('workflowVersions')
|
|
440
|
-
.select(['workflowName', 'graphHash', 'graph'])
|
|
441
|
-
.where('source', '=', 'dynamic-workflow')
|
|
442
|
-
.where('status', '=', 'active');
|
|
443
|
-
if (agentName) {
|
|
444
|
-
query = query.where('workflowName', 'like', `ai:${agentName}:%`);
|
|
445
|
-
}
|
|
446
|
-
const rows = await query.execute();
|
|
447
|
-
return rows.map((row) => ({
|
|
448
|
-
workflowName: row.workflowName,
|
|
449
|
-
graphHash: row.graphHash,
|
|
450
|
-
graph: typeof row.graph === 'string' ? JSON.parse(row.graph) : row.graph,
|
|
451
|
-
}));
|
|
452
|
-
}
|
|
453
513
|
async close() { }
|
|
454
514
|
}
|
|
@@ -51,6 +51,23 @@ export declare const unsatisfiedRequirements: (db: Kysely<any>, schemas?: PikkuS
|
|
|
51
51
|
* is the engine's to give, not ours.
|
|
52
52
|
*/
|
|
53
53
|
export declare const applyPikkuSchemas: (db: Kysely<any>, schemas?: PikkuSchema[]) => Promise<void>;
|
|
54
|
+
/** A table a schema creates, as the DDL addresses it. */
|
|
55
|
+
export interface DeclaredTable {
|
|
56
|
+
/** Present when the connection is `withSchema(...)`-bound. */
|
|
57
|
+
schema?: string;
|
|
58
|
+
name: string;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* The tables a schema creates, read back out of its own compiled SQL.
|
|
62
|
+
*
|
|
63
|
+
* Derived rather than declared alongside the statements, because a hand-kept
|
|
64
|
+
* list of names is a second source of truth: rename a table in a statement and
|
|
65
|
+
* the list keeps answering for the old one.
|
|
66
|
+
*
|
|
67
|
+
* Compiled against `db` rather than in the abstract, so the answer carries the
|
|
68
|
+
* schema the DDL will actually target — the same binding the lookup has to use.
|
|
69
|
+
*/
|
|
70
|
+
export declare const declaredTables: (schema: PikkuSchema, db: Kysely<any>) => DeclaredTable[];
|
|
54
71
|
/** What `ensurePikkuSchema` found when it looked. */
|
|
55
72
|
export type EnsureOutcome = 'present' | 'created';
|
|
56
73
|
/**
|
package/dist/src/schema/index.js
CHANGED
|
@@ -103,29 +103,113 @@ export const applyPikkuSchemas = async (db, schemas = pikkuSchemas) => {
|
|
|
103
103
|
const bound = bind(trx);
|
|
104
104
|
for (const schema of schemas) {
|
|
105
105
|
for (const statement of schema.statements) {
|
|
106
|
-
|
|
106
|
+
const query = statement(bound, types);
|
|
107
|
+
try {
|
|
108
|
+
await query.execute();
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
throw explain(error, query.compile().sql, schema);
|
|
112
|
+
}
|
|
107
113
|
}
|
|
108
114
|
}
|
|
109
115
|
});
|
|
110
116
|
};
|
|
117
|
+
/** `references "app"."workflow_runs"` — a foreign key onto a qualified table. */
|
|
118
|
+
const QUALIFIED_REFERENCE = /references "[^"]+"\."[^"]+"/i;
|
|
119
|
+
/**
|
|
120
|
+
* sqlite's own words for "you put a dot where a table name goes".
|
|
121
|
+
*
|
|
122
|
+
* Matched rather than a driver's `code`, because every sqlite binding
|
|
123
|
+
* (better-sqlite3, node:sqlite, bun:sqlite, libsql) passes the engine's message
|
|
124
|
+
* through verbatim while disagreeing about how to label it. Postgres phrases the
|
|
125
|
+
* same class of error the other way round — `syntax error at or near "."` — so
|
|
126
|
+
* this cannot fire on it.
|
|
127
|
+
*/
|
|
128
|
+
const SQLITE_REJECTED_QUALIFIER = /near "\.":\s*syntax error/i;
|
|
129
|
+
/**
|
|
130
|
+
* Say what a schema-bound connection did to the DDL, when that is what failed.
|
|
131
|
+
*
|
|
132
|
+
* `withSchema(...)` qualifies foreign key targets along with everything else.
|
|
133
|
+
* Postgres needs that — an unqualified reference resolves against `search_path`,
|
|
134
|
+
* which a schema-bound connection does not control — and sqlite refuses it: a
|
|
135
|
+
* `REFERENCES` clause there takes a bare table name, because a foreign key can
|
|
136
|
+
* only point inside the same database, so there is no qualifier to give. One
|
|
137
|
+
* declaration cannot spell both, and what the engine says about it is
|
|
138
|
+
* `near ".": syntax error`, which names neither the schema nor the cause.
|
|
139
|
+
*
|
|
140
|
+
* Gated on sqlite's exact phrasing as well as the SQL. Postgres compiles the
|
|
141
|
+
* same qualified reference legitimately, and can raise a syntax error of its own
|
|
142
|
+
* on this DDL — a column type resolved from `requires` goes in verbatim — so a
|
|
143
|
+
* looser `/syntax error/` would answer for postgres in sqlite's name.
|
|
144
|
+
*/
|
|
145
|
+
const explain = (error, sql, schema) => {
|
|
146
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
147
|
+
if (!QUALIFIED_REFERENCE.test(sql) ||
|
|
148
|
+
!SQLITE_REJECTED_QUALIFIER.test(message)) {
|
|
149
|
+
return error;
|
|
150
|
+
}
|
|
151
|
+
return new Error(`The '${schema.name}' schema has a foreign key, and this connection is bound to a schema, ` +
|
|
152
|
+
`so it compiled to a qualified 'references "schema"."table"' that the database rejected. ` +
|
|
153
|
+
'sqlite is the engine that does this: its REFERENCES clause takes a bare table name. ' +
|
|
154
|
+
'Use a connection without withSchema(...) there — sqlite has one schema to be in — ' +
|
|
155
|
+
'or write the schema down as a migration with `pikku db generate` and apply it yourself. ' +
|
|
156
|
+
`The database said: ${message}`, { cause: error });
|
|
157
|
+
};
|
|
158
|
+
/** How a table reads in an error message: qualified only when it is. */
|
|
159
|
+
const displayName = (table) => table.schema ? `${table.schema}.${table.name}` : table.name;
|
|
160
|
+
/**
|
|
161
|
+
* `create table "workflow_runs"` and `create table "app"."workflow_runs"`.
|
|
162
|
+
*
|
|
163
|
+
* The schema half is optional and the table is always the LAST quoted
|
|
164
|
+
* identifier — matching the first one instead reads a `withSchema('app')`
|
|
165
|
+
* connection's DDL as a table called `app`, which is no table at all.
|
|
166
|
+
*/
|
|
167
|
+
const CREATE_TABLE = /^\s*create table (?:if not exists\s+)?(?:"([^"]+)"\.)?"([^"]+)"/i;
|
|
111
168
|
/**
|
|
112
169
|
* The tables a schema creates, read back out of its own compiled SQL.
|
|
113
170
|
*
|
|
114
171
|
* Derived rather than declared alongside the statements, because a hand-kept
|
|
115
172
|
* list of names is a second source of truth: rename a table in a statement and
|
|
116
173
|
* the list keeps answering for the old one.
|
|
174
|
+
*
|
|
175
|
+
* Compiled against `db` rather than in the abstract, so the answer carries the
|
|
176
|
+
* schema the DDL will actually target — the same binding the lookup has to use.
|
|
117
177
|
*/
|
|
118
|
-
const declaredTables = (schema, db) => {
|
|
178
|
+
export const declaredTables = (schema, db) => {
|
|
119
179
|
const bound = bind(db);
|
|
120
180
|
const tables = [];
|
|
121
181
|
for (const statement of schema.statements) {
|
|
122
|
-
const match =
|
|
182
|
+
const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql);
|
|
123
183
|
if (match)
|
|
124
|
-
tables.push(match[1]);
|
|
184
|
+
tables.push({ schema: match[1], name: match[2] });
|
|
125
185
|
}
|
|
126
186
|
return tables;
|
|
127
187
|
};
|
|
128
|
-
const tablesOf = async (db) =>
|
|
188
|
+
const tablesOf = async (db) => {
|
|
189
|
+
const tables = await bind(db).introspection.getTables();
|
|
190
|
+
return {
|
|
191
|
+
qualified: new Set(tables
|
|
192
|
+
.filter((table) => table.schema)
|
|
193
|
+
.map((table) => `${table.schema}.${table.name}`)),
|
|
194
|
+
bare: new Set(tables.map((table) => table.name)),
|
|
195
|
+
schemaAware: tables.some((table) => table.schema !== undefined),
|
|
196
|
+
};
|
|
197
|
+
};
|
|
198
|
+
/**
|
|
199
|
+
* Qualified declarations match on the pair — but only where that pair means
|
|
200
|
+
* something.
|
|
201
|
+
*
|
|
202
|
+
* `TableMetadata.schema` is optional, and sqlite and mysql leave it unset:
|
|
203
|
+
* sqlite has one schema to be in, and mysql calls the thing a database. A
|
|
204
|
+
* `withSchema('main')` connection there declares `main.workflow_runs` while
|
|
205
|
+
* introspection can only offer `workflow_runs`, and holding out for the pair
|
|
206
|
+
* would read every existing table as missing — the same false negative this
|
|
207
|
+
* whole function exists to fix, arrived at from the other side. So when the
|
|
208
|
+
* engine reports no schemas, the bare name is the only name there is.
|
|
209
|
+
*/
|
|
210
|
+
const isPresent = (existing, table) => table.schema && existing.schemaAware
|
|
211
|
+
? existing.qualified.has(`${table.schema}.${table.name}`)
|
|
212
|
+
: existing.bare.has(table.name);
|
|
129
213
|
/**
|
|
130
214
|
* Make sure one schema's tables are there, creating them only if none are.
|
|
131
215
|
*
|
|
@@ -147,12 +231,15 @@ const tablesOf = async (db) => new Set((await bind(db).introspection.getTables()
|
|
|
147
231
|
export const ensurePikkuSchema = async (db, schema) => {
|
|
148
232
|
const existing = await tablesOf(db);
|
|
149
233
|
const declared = declaredTables(schema, db);
|
|
150
|
-
const missing = declared.filter((table) => !existing
|
|
234
|
+
const missing = declared.filter((table) => !isPresent(existing, table));
|
|
151
235
|
if (missing.length === 0)
|
|
152
236
|
return 'present';
|
|
153
237
|
if (missing.length < declared.length) {
|
|
154
|
-
throw new Error(`The '${schema.name}' schema is half applied: ${missing.join(', ')} missing, ` +
|
|
155
|
-
`${declared
|
|
238
|
+
throw new Error(`The '${schema.name}' schema is half applied: ${missing.map(displayName).join(', ')} missing, ` +
|
|
239
|
+
`${declared
|
|
240
|
+
.filter((table) => isPresent(existing, table))
|
|
241
|
+
.map(displayName)
|
|
242
|
+
.join(', ')} already there. ` +
|
|
156
243
|
'Something else owns part of it. Reconcile it with a migration — ' +
|
|
157
244
|
'`pikku db generate` writes the declaration down — rather than creating the rest at boot.');
|
|
158
245
|
}
|
|
@@ -167,7 +254,7 @@ export const ensurePikkuSchema = async (db, schema) => {
|
|
|
167
254
|
// a guarantee — a boot race is only fully answered by having run the
|
|
168
255
|
// migration, which is what `pikku db generate` is for.
|
|
169
256
|
const after = await tablesOf(db);
|
|
170
|
-
if (declared.every((table) => after
|
|
257
|
+
if (declared.every((table) => isPresent(after, table)))
|
|
171
258
|
return 'present';
|
|
172
259
|
throw error;
|
|
173
260
|
}
|
|
@@ -6,8 +6,10 @@ import type { PikkuSchema } from './pikku-schema.types.js';
|
|
|
6
6
|
* primary keys had `defaultTo(sql.raw("'" + crypto.randomUUID() + "'"))`, which
|
|
7
7
|
* evaluates once while the statement is built — every row would have taken the
|
|
8
8
|
* same default, so it was never a generator; the services supply the id. And
|
|
9
|
-
* `workflowStep.fromStepName
|
|
10
|
-
*
|
|
9
|
+
* `workflowStep.fromStepName`, `workflowStep.currentAttempt` and
|
|
10
|
+
* `workflowStepHistory.attempt` were each backfilled by an
|
|
11
|
+
* `alterTable(...).catch(() => {})` bolted on after the fact, which is the job a
|
|
12
|
+
* migration does — they are declared here and nowhere else.
|
|
11
13
|
*
|
|
12
14
|
* `KyselyWorkflowMirror` created these same four tables separately. There is one
|
|
13
15
|
* declaration now, and both services read it.
|
|
@@ -6,8 +6,10 @@ import { sql } from 'kysely';
|
|
|
6
6
|
* primary keys had `defaultTo(sql.raw("'" + crypto.randomUUID() + "'"))`, which
|
|
7
7
|
* evaluates once while the statement is built — every row would have taken the
|
|
8
8
|
* same default, so it was never a generator; the services supply the id. And
|
|
9
|
-
* `workflowStep.fromStepName
|
|
10
|
-
*
|
|
9
|
+
* `workflowStep.fromStepName`, `workflowStep.currentAttempt` and
|
|
10
|
+
* `workflowStepHistory.attempt` were each backfilled by an
|
|
11
|
+
* `alterTable(...).catch(() => {})` bolted on after the fact, which is the job a
|
|
12
|
+
* migration does — they are declared here and nowhere else.
|
|
11
13
|
*
|
|
12
14
|
* `KyselyWorkflowMirror` created these same four tables separately. There is one
|
|
13
15
|
* declaration now, and both services read it.
|
|
@@ -49,6 +51,7 @@ export const workflowSchema = {
|
|
|
49
51
|
.addColumn('retries', 'integer')
|
|
50
52
|
.addColumn('retryDelay', 'text')
|
|
51
53
|
.addColumn('fromStepName', 'text')
|
|
54
|
+
.addColumn('currentAttempt', 'integer')
|
|
52
55
|
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
53
56
|
.addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
54
57
|
.addUniqueConstraint('workflow_step_run_name_unique', [
|
|
@@ -65,6 +68,7 @@ export const workflowSchema = {
|
|
|
65
68
|
.addColumn('status', 'text', (col) => col.notNull())
|
|
66
69
|
.addColumn('result', 'text')
|
|
67
70
|
.addColumn('error', 'text')
|
|
71
|
+
.addColumn('attempt', 'integer')
|
|
68
72
|
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
69
73
|
.addColumn('runningAt', 'timestamp')
|
|
70
74
|
.addColumn('scheduledAt', 'timestamp')
|
|
@@ -82,5 +86,27 @@ export const workflowSchema = {
|
|
|
82
86
|
'workflowName',
|
|
83
87
|
'graphHash',
|
|
84
88
|
]),
|
|
89
|
+
// The indexes trail the tables so that a table is never indexed before it
|
|
90
|
+
// exists, whatever order the declarations are read in.
|
|
91
|
+
(db) => db.schema
|
|
92
|
+
.createIndex('idx_workflow_runs_status_created')
|
|
93
|
+
.on('workflowRuns')
|
|
94
|
+
.columns(['status', 'createdAt']),
|
|
95
|
+
(db) => db.schema
|
|
96
|
+
.createIndex('idx_workflow_runs_workflow_created')
|
|
97
|
+
.on('workflowRuns')
|
|
98
|
+
.columns(['workflow', 'createdAt']),
|
|
99
|
+
(db) => db.schema
|
|
100
|
+
.createIndex('idx_workflow_step_run_status')
|
|
101
|
+
.on('workflowStep')
|
|
102
|
+
.columns(['workflowRunId', 'status']),
|
|
103
|
+
(db) => db.schema
|
|
104
|
+
.createIndex('idx_workflow_step_history_step')
|
|
105
|
+
.on('workflowStepHistory')
|
|
106
|
+
.columns(['workflowStepId', 'createdAt']),
|
|
107
|
+
(db) => db.schema
|
|
108
|
+
.createIndex('idx_workflow_versions_source_status')
|
|
109
|
+
.on('workflowVersions')
|
|
110
|
+
.columns(['source', 'status']),
|
|
85
111
|
],
|
|
86
112
|
};
|