@pikku/kysely 0.13.18 → 0.13.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/kysely-agent-run-state-service.js +2 -2
- package/dist/src/kysely-agent-storage-service.js +2 -2
- package/dist/src/kysely-audit-service.js +2 -2
- package/dist/src/kysely-channel-store.js +2 -2
- package/dist/src/kysely-credential-service.js +2 -2
- package/dist/src/kysely-deployment-service.js +2 -2
- package/dist/src/kysely-scope-service.js +2 -2
- package/dist/src/kysely-secret-service.js +2 -2
- package/dist/src/kysely-session-store.js +2 -2
- package/dist/src/kysely-tables.d.ts +28 -0
- package/dist/src/kysely-virtual-user-run-store.js +2 -2
- package/dist/src/kysely-webhook-service.js +2 -2
- package/dist/src/kysely-workflow-mirror.d.ts +1 -1
- package/dist/src/kysely-workflow-mirror.js +3 -3
- package/dist/src/kysely-workflow-service.d.ts +39 -8
- package/dist/src/kysely-workflow-service.js +77 -11
- package/dist/src/schema/agent.schema.js +1 -0
- package/dist/src/schema/audit.schema.d.ts +5 -4
- package/dist/src/schema/audit.schema.js +6 -4
- package/dist/src/schema/channel.schema.js +1 -0
- package/dist/src/schema/credential.schema.js +1 -0
- package/dist/src/schema/index.d.ts +47 -26
- package/dist/src/schema/index.js +56 -46
- package/dist/src/schema/pikku-schema.types.d.ts +20 -0
- package/dist/src/schema/scope.schema.js +1 -0
- package/dist/src/schema/virtual-user.schema.js +2 -1
- package/dist/src/schema/webhook.schema.js +1 -0
- package/dist/src/schema/workflow.schema.js +1 -0
- package/package.json +2 -2
- package/src/index.ts +12 -2
- package/src/kysely-agent-run-state-service.ts +2 -2
- package/src/kysely-agent-storage-service.ts +2 -2
- package/src/kysely-audit-service.test.ts +2 -10
- package/src/kysely-audit-service.ts +2 -2
- package/src/kysely-channel-store.ts +2 -2
- package/src/kysely-credential-service.ts +2 -2
- package/src/kysely-deployment-service.ts +2 -2
- package/src/kysely-scope-service.test.ts +2 -0
- package/src/kysely-scope-service.ts +2 -2
- package/src/kysely-secret-service.ts +2 -2
- package/src/kysely-services.test.ts +26 -0
- package/src/kysely-session-store.ts +2 -2
- package/src/kysely-tables.test.ts +1 -12
- package/src/kysely-tables.ts +29 -0
- package/src/kysely-virtual-user-run-store.test.ts +22 -6
- package/src/kysely-virtual-user-run-store.ts +2 -2
- package/src/kysely-webhook-service.test.ts +4 -0
- package/src/kysely-webhook-service.ts +2 -2
- package/src/kysely-workflow-mirror.test.ts +2 -0
- package/src/kysely-workflow-mirror.ts +3 -3
- package/src/kysely-workflow-service.test.ts +114 -0
- package/src/kysely-workflow-service.ts +99 -11
- package/src/schema/agent.schema.ts +1 -0
- package/src/schema/audit.schema.ts +6 -4
- package/src/schema/channel.schema.ts +1 -0
- package/src/schema/credential.schema.ts +1 -0
- package/src/schema/index.ts +67 -52
- package/src/schema/pikku-schema.types.ts +20 -0
- package/src/schema/schema.test.ts +86 -29
- package/src/schema/scope.schema.ts +1 -0
- package/src/schema/virtual-user.schema.ts +2 -1
- package/src/schema/webhook.schema.ts +1 -0
- package/src/schema/workflow.schema.ts +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,80 @@
|
|
|
1
|
+
## 0.13.20
|
|
2
|
+
|
|
3
|
+
### Patch Changes
|
|
4
|
+
|
|
5
|
+
- 274cab3: The singleton intersection moves to the leaves that name it, the runtime stops
|
|
6
|
+
creating schema, and `db generate` writes only the runtime tables a project's
|
|
7
|
+
services own
|
|
8
|
+
|
|
9
|
+
`WiredSingletonServices` was exported from the generated function leaf so the
|
|
10
|
+
auth leaf could import it. Nothing outside a generated leaf ever names it —
|
|
11
|
+
emit declarations for a project of any size and it appears in none of them —
|
|
12
|
+
so the auth and middleware leaves derive the intersection themselves and the
|
|
13
|
+
function leaf keeps it private. `WiredServices` stays exported: 147 `.d.ts`
|
|
14
|
+
files name it, and unexporting it asks every wired module to name each member
|
|
15
|
+
service through a specifier it does not have.
|
|
16
|
+
|
|
17
|
+
`ensurePikkuSchema` is gone. `requirePikkuSchema` replaces it: a service calls
|
|
18
|
+
it at boot, it looks, and it issues no DDL at all. `pikku db generate` writes
|
|
19
|
+
the declaration down as a migration and `pikku db migrate` applies it, and
|
|
20
|
+
those two are now the only way pikku's runtime tables come into existence. A
|
|
21
|
+
service that finds them missing says so and stops, naming both commands.
|
|
22
|
+
Half-present is no longer a distinct case — the remedy is the same migration
|
|
23
|
+
either way. `audit` and `virtual-user` join `pikkuSchemas` as a consequence:
|
|
24
|
+
boot was the only thing that had ever created them.
|
|
25
|
+
|
|
26
|
+
`pikku db generate` applied all of `pikkuSchemas`, so a project with no agents,
|
|
27
|
+
no channels and no workflows still had `agent_threads`, `channels` and
|
|
28
|
+
`workflow_runs` written into its migrations, and then carried them forever. A
|
|
29
|
+
schema now names the services that own it, and generation gates on
|
|
30
|
+
`requiredServices` — the set the inspector already builds for service
|
|
31
|
+
tree-shaking. The gate is one-sided: a schema that names no owner is always
|
|
32
|
+
written, because the session and secret stores and the deployment record are
|
|
33
|
+
reached by the runtime itself and nothing in a project's source implies them.
|
|
34
|
+
Declared scopes now imply `scopeService`, which nothing destructures because
|
|
35
|
+
the generated auth layer is what reaches it.
|
|
36
|
+
|
|
37
|
+
Drift keeps asking the unscoped question. A table already in a database has to
|
|
38
|
+
stay recognisable as a runtime table after the service that needed it is
|
|
39
|
+
dropped — scoping it there would report those tables as unexplained.
|
|
40
|
+
|
|
41
|
+
Every project in this repo that had been relying on boot-time creation now says
|
|
42
|
+
where its tables come from. `createConfig` moves into its own `config.ts` in the
|
|
43
|
+
templates — `pikku db` looks for it there — and the three postgres templates plus
|
|
44
|
+
the workflow verifier declare `postgresUrl` and run `pikku db generate && pikku
|
|
45
|
+
db migrate` before the server starts, from the single connection string their
|
|
46
|
+
runtime opens. The e2e harness cannot: its databases are in-memory sqlite built
|
|
47
|
+
inside the services factory, so nothing outside the process can migrate them. It
|
|
48
|
+
applies the schemas it owns with `applyPikkuSchemas` instead — the same DDL, run
|
|
49
|
+
by the one process that has the database.
|
|
50
|
+
|
|
51
|
+
- Updated dependencies [32616af]
|
|
52
|
+
- Updated dependencies [6848cd9]
|
|
53
|
+
- @pikku/core@0.12.89
|
|
54
|
+
|
|
55
|
+
## 0.13.19
|
|
56
|
+
|
|
57
|
+
### Patch Changes
|
|
58
|
+
|
|
59
|
+
- 2d21628: fix(kysely): claim a workflow step atomically in every SQL dialect
|
|
60
|
+
|
|
61
|
+
The workflow engine's "atomic claim" was a read-then-write guarded by
|
|
62
|
+
`withStepLock`, and `@pikku/kysely` inherited a silent pass-through for that
|
|
63
|
+
lock — so on every dialect but Postgres and MySQL a redelivered queue job could
|
|
64
|
+
claim a step another dispatch was already running, executing a side-effecting
|
|
65
|
+
step twice.
|
|
66
|
+
|
|
67
|
+
`@pikku/kysely` now claims the step with a status-guarded `UPDATE` and reads the
|
|
68
|
+
affected-row count, which is atomic in every SQL dialect without an
|
|
69
|
+
advisory-lock primitive. Relay redispatch is enabled for all Kysely dialects as
|
|
70
|
+
a result, not just Postgres and MySQL.
|
|
71
|
+
|
|
72
|
+
- Updated dependencies [9687ad1]
|
|
73
|
+
- Updated dependencies [2d21628]
|
|
74
|
+
- Updated dependencies [985b87b]
|
|
75
|
+
- Updated dependencies [3a83f85]
|
|
76
|
+
- @pikku/core@0.12.87
|
|
77
|
+
|
|
1
78
|
## 0.13.18
|
|
2
79
|
|
|
3
80
|
### Patch Changes
|
package/dist/src/index.d.ts
CHANGED
|
@@ -17,7 +17,7 @@ export { KyselyAuditService } from './kysely-audit-service.js';
|
|
|
17
17
|
export { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js';
|
|
18
18
|
export { SerializePlugin, BaseSerializePlugin, type Serializer, type Deserializer, } from './serialize-plugin.js';
|
|
19
19
|
export { createCoercionPlugin, type ColumnKind, type CoercionMap, type CreateCoercionPluginOptions, } from './coercion-plugin.js';
|
|
20
|
-
export { auditSchema, pikkuSchemas, applyPikkuSchemas, compilePikkuSchemas,
|
|
20
|
+
export { agentSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, type PikkuSchema, type RequiredTypes, type UnmetRequirement, type SchemaStatementFactory, } from './schema/index.js';
|
|
21
21
|
export type { KyselyPikkuDB } from './kysely-tables.js';
|
|
22
22
|
export type { WorkflowRunService } from '@pikku/core/workflow';
|
|
23
23
|
export type { AgentRunService, AgentRunRow } from '@pikku/core/agent';
|
package/dist/src/index.js
CHANGED
|
@@ -17,4 +17,4 @@ export { KyselyAuditService } from './kysely-audit-service.js';
|
|
|
17
17
|
export { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js';
|
|
18
18
|
export { SerializePlugin, BaseSerializePlugin, } from './serialize-plugin.js';
|
|
19
19
|
export { createCoercionPlugin, } from './coercion-plugin.js';
|
|
20
|
-
export { auditSchema, pikkuSchemas, applyPikkuSchemas, compilePikkuSchemas,
|
|
20
|
+
export { agentSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, } from './schema/index.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { sql } from 'kysely';
|
|
2
|
-
import {
|
|
2
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
3
3
|
import { agentSchema } from './schema/agent.schema.js';
|
|
4
4
|
import { getRunScores, saveRunScore } from './kysely-agent-run-scores.js';
|
|
5
5
|
export class KyselyAgentRunStateService {
|
|
@@ -11,7 +11,7 @@ export class KyselyAgentRunStateService {
|
|
|
11
11
|
async init() {
|
|
12
12
|
if (this.initialized)
|
|
13
13
|
return;
|
|
14
|
-
await
|
|
14
|
+
await requirePikkuSchema(this.db, agentSchema);
|
|
15
15
|
this.initialized = true;
|
|
16
16
|
}
|
|
17
17
|
async createRun(run) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseJson } from './kysely-json.js';
|
|
2
2
|
import { getRunScores, saveRunScore } from './kysely-agent-run-scores.js';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { agentSchema } from './schema/agent.schema.js';
|
|
5
5
|
export class KyselyAgentStorageService {
|
|
6
6
|
db;
|
|
@@ -11,7 +11,7 @@ export class KyselyAgentStorageService {
|
|
|
11
11
|
async init() {
|
|
12
12
|
if (this.initialized)
|
|
13
13
|
return;
|
|
14
|
-
await
|
|
14
|
+
await requirePikkuSchema(this.db, agentSchema);
|
|
15
15
|
this.initialized = true;
|
|
16
16
|
}
|
|
17
17
|
async createThread(resourceId, options) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
2
2
|
import { auditSchema } from './schema/audit.schema.js';
|
|
3
3
|
const jsonOrNull = (v) => v != null ? JSON.stringify(v) : null;
|
|
4
4
|
const parseJson = (v) => {
|
|
@@ -90,7 +90,7 @@ export class KyselyAuditService {
|
|
|
90
90
|
async init() {
|
|
91
91
|
if (this.initialized)
|
|
92
92
|
return;
|
|
93
|
-
await
|
|
93
|
+
await requirePikkuSchema(this.db, auditSchema);
|
|
94
94
|
this.initialized = true;
|
|
95
95
|
}
|
|
96
96
|
async audit(event) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ChannelStore } from '@pikku/core/channel';
|
|
2
2
|
import { parseJson } from './kysely-json.js';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { channelSchema } from './schema/channel.schema.js';
|
|
5
5
|
export class KyselyChannelStore extends ChannelStore {
|
|
6
6
|
db;
|
|
@@ -12,7 +12,7 @@ export class KyselyChannelStore extends ChannelStore {
|
|
|
12
12
|
async init() {
|
|
13
13
|
if (this.initialized)
|
|
14
14
|
return;
|
|
15
|
-
await
|
|
15
|
+
await requirePikkuSchema(this.db, channelSchema);
|
|
16
16
|
this.initialized = true;
|
|
17
17
|
}
|
|
18
18
|
async addChannel({ channelId, channelName, openingData, }) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { deriveKEK, envelopeDecrypt, envelopeEncrypt, envelopeRewrap, generateKEKSalt, } from '@pikku/core/crypto-utils';
|
|
2
|
-
import {
|
|
2
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
3
3
|
import { credentialSchema } from './schema/credential.schema.js';
|
|
4
4
|
export class KyselyCredentialService {
|
|
5
5
|
db;
|
|
@@ -22,7 +22,7 @@ export class KyselyCredentialService {
|
|
|
22
22
|
async init() {
|
|
23
23
|
if (this.initialized)
|
|
24
24
|
return;
|
|
25
|
-
await
|
|
25
|
+
await requirePikkuSchema(this.db, credentialSchema);
|
|
26
26
|
this.initialized = true;
|
|
27
27
|
}
|
|
28
28
|
async logAudit(name, userId, action) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { buildRemoteHeaders } from '@pikku/core/remote';
|
|
2
2
|
import { getAllFunctionNames } from '@pikku/core/function';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { deploymentSchema } from './schema/deployment.schema.js';
|
|
5
5
|
export class KyselyDeploymentService {
|
|
6
6
|
db;
|
|
@@ -21,7 +21,7 @@ export class KyselyDeploymentService {
|
|
|
21
21
|
async init() {
|
|
22
22
|
if (this.initialized)
|
|
23
23
|
return;
|
|
24
|
-
await
|
|
24
|
+
await requirePikkuSchema(this.db, deploymentSchema);
|
|
25
25
|
this.initialized = true;
|
|
26
26
|
}
|
|
27
27
|
async start(config) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SystemRoleImmutableError, SystemRoleShadowedError, } from '@pikku/core/errors';
|
|
2
|
-
import {
|
|
2
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
3
3
|
import { scopeSchema } from './schema/scope.schema.js';
|
|
4
4
|
/**
|
|
5
5
|
* Resolves and administers user scopes against four self-created tables.
|
|
@@ -22,7 +22,7 @@ export class KyselyScopeService {
|
|
|
22
22
|
async init() {
|
|
23
23
|
if (this.initialized)
|
|
24
24
|
return;
|
|
25
|
-
await
|
|
25
|
+
await requirePikkuSchema(this.db, scopeSchema);
|
|
26
26
|
this.initialized = true;
|
|
27
27
|
}
|
|
28
28
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { deriveKEK, envelopeDecrypt, envelopeEncrypt, envelopeRewrap, generateKEKSalt, } from '@pikku/core/crypto-utils';
|
|
2
2
|
import { createSecretValue, isSecretValue } from '@pikku/core/classification';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { secretSchema } from './schema/secret.schema.js';
|
|
5
5
|
export class KyselySecretService {
|
|
6
6
|
db;
|
|
@@ -23,7 +23,7 @@ export class KyselySecretService {
|
|
|
23
23
|
async init() {
|
|
24
24
|
if (this.initialized)
|
|
25
25
|
return;
|
|
26
|
-
await
|
|
26
|
+
await requirePikkuSchema(this.db, secretSchema);
|
|
27
27
|
this.initialized = true;
|
|
28
28
|
}
|
|
29
29
|
async logAudit(secretKey, action) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { parseJson } from './kysely-json.js';
|
|
2
|
-
import {
|
|
2
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
3
3
|
import { sessionSchema } from './schema/session.schema.js';
|
|
4
4
|
export class KyselySessionStore {
|
|
5
5
|
db;
|
|
@@ -10,7 +10,7 @@ export class KyselySessionStore {
|
|
|
10
10
|
async init() {
|
|
11
11
|
if (this.initialized)
|
|
12
12
|
return;
|
|
13
|
-
await
|
|
13
|
+
await requirePikkuSchema(this.db, sessionSchema);
|
|
14
14
|
this.initialized = true;
|
|
15
15
|
}
|
|
16
16
|
async get(pikkuUserId) {
|
|
@@ -289,6 +289,33 @@ export interface VirtualUserRunTable {
|
|
|
289
289
|
createdAt: Generated<Date | string>;
|
|
290
290
|
finishedAt: Date | string | null;
|
|
291
291
|
}
|
|
292
|
+
/**
|
|
293
|
+
* One audit event. Every column is text on every engine so a locally-run
|
|
294
|
+
* project and a deployed stage write rows the same reader can read; the JSON
|
|
295
|
+
* columns (`tables`, `changedCols`, `event`, `old`, `data`) are serialised by
|
|
296
|
+
* {@link KyselyAuditService} rather than by a plugin.
|
|
297
|
+
*/
|
|
298
|
+
export interface AuditTable {
|
|
299
|
+
auditId: string;
|
|
300
|
+
/** ISO 8601, because string ordering is chronological ordering. */
|
|
301
|
+
occurredAt: string;
|
|
302
|
+
type: string;
|
|
303
|
+
source: Generated<string>;
|
|
304
|
+
outcome: string | null;
|
|
305
|
+
functionId: string | null;
|
|
306
|
+
wireType: string | null;
|
|
307
|
+
traceId: string | null;
|
|
308
|
+
transactionId: string | null;
|
|
309
|
+
queryId: string | null;
|
|
310
|
+
userId: string | null;
|
|
311
|
+
orgId: string | null;
|
|
312
|
+
pikkuUserId: string | null;
|
|
313
|
+
tables: string | null;
|
|
314
|
+
changedCols: string | null;
|
|
315
|
+
event: string | null;
|
|
316
|
+
old: string | null;
|
|
317
|
+
data: string | null;
|
|
318
|
+
}
|
|
292
319
|
export interface KyselyPikkuDB {
|
|
293
320
|
pikkuScopes: PikkuScopesTable;
|
|
294
321
|
pikkuRoles: PikkuRolesTable;
|
|
@@ -319,4 +346,5 @@ export interface KyselyPikkuDB {
|
|
|
319
346
|
webhookDelivery: WebhookDeliveryTable;
|
|
320
347
|
webhookDeliveryAttempt: WebhookDeliveryAttemptTable;
|
|
321
348
|
virtualUserRun: VirtualUserRunTable;
|
|
349
|
+
audit: AuditTable;
|
|
322
350
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { parseJson } from './kysely-json.js';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { virtualUserSchema } from './schema/virtual-user.schema.js';
|
|
5
5
|
/**
|
|
6
6
|
* Records virtual-user runs in a `virtualUserRun` table.
|
|
@@ -35,7 +35,7 @@ export class KyselyVirtualUserRunStore {
|
|
|
35
35
|
async init() {
|
|
36
36
|
if (this.initialized)
|
|
37
37
|
return;
|
|
38
|
-
await
|
|
38
|
+
await requirePikkuSchema(this.db, virtualUserSchema);
|
|
39
39
|
this.initialized = true;
|
|
40
40
|
}
|
|
41
41
|
async start(run) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { QueueWebhookService } from '@pikku/core/services';
|
|
2
2
|
import { PIKKU_OUTGOING_WEBHOOK_QUEUE_NAME, } from '@pikku/core/services';
|
|
3
|
-
import {
|
|
3
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
4
4
|
import { webhookSchema } from './schema/webhook.schema.js';
|
|
5
5
|
/**
|
|
6
6
|
* Durable {@link QueueWebhookService}: still delivers through the
|
|
@@ -19,7 +19,7 @@ export class KyselyWebhookService extends QueueWebhookService {
|
|
|
19
19
|
async init() {
|
|
20
20
|
if (this.initialized)
|
|
21
21
|
return;
|
|
22
|
-
await
|
|
22
|
+
await requirePikkuSchema(this.db, webhookSchema);
|
|
23
23
|
this.initialized = true;
|
|
24
24
|
}
|
|
25
25
|
async send(input) {
|
|
@@ -22,7 +22,7 @@ export declare class KyselyWorkflowMirror implements WorkflowRunMirror {
|
|
|
22
22
|
* Create the underlying tables if none of them exist yet.
|
|
23
23
|
*
|
|
24
24
|
* Safe to call from either the mirror or `KyselyWorkflowService.init()` —
|
|
25
|
-
* both apply the same `workflowSchema`, and `
|
|
25
|
+
* both apply the same `workflowSchema`, and `requirePikkuSchema` is a no-op
|
|
26
26
|
* once the tables are there. A database holding only *part* of the schema
|
|
27
27
|
* throws rather than filling in the rest: something else owns those tables,
|
|
28
28
|
* and boot is not where that gets reconciled.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
2
2
|
import { workflowSchema } from './schema/workflow.schema.js';
|
|
3
3
|
/**
|
|
4
4
|
* Kysely-backed `WorkflowRunMirror`.
|
|
@@ -22,7 +22,7 @@ export class KyselyWorkflowMirror {
|
|
|
22
22
|
* Create the underlying tables if none of them exist yet.
|
|
23
23
|
*
|
|
24
24
|
* Safe to call from either the mirror or `KyselyWorkflowService.init()` —
|
|
25
|
-
* both apply the same `workflowSchema`, and `
|
|
25
|
+
* both apply the same `workflowSchema`, and `requirePikkuSchema` is a no-op
|
|
26
26
|
* once the tables are there. A database holding only *part* of the schema
|
|
27
27
|
* throws rather than filling in the rest: something else owns those tables,
|
|
28
28
|
* and boot is not where that gets reconciled.
|
|
@@ -30,7 +30,7 @@ export class KyselyWorkflowMirror {
|
|
|
30
30
|
async init() {
|
|
31
31
|
if (this.initialized)
|
|
32
32
|
return;
|
|
33
|
-
await
|
|
33
|
+
await requirePikkuSchema(this.db, workflowSchema);
|
|
34
34
|
this.initialized = true;
|
|
35
35
|
}
|
|
36
36
|
async createRun(runId, workflowName, input, inline, graphHash, wire, options) {
|
|
@@ -47,7 +47,36 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
47
47
|
protected setStepErrorImpl(stepId: string, error: Error): Promise<void>;
|
|
48
48
|
protected createRetryAttemptImpl(stepId: string, status: 'pending' | 'running'): Promise<StepState>;
|
|
49
49
|
withRunLock<T>(_id: string, fn: () => Promise<T>): Promise<T>;
|
|
50
|
+
/**
|
|
51
|
+
* A pass-through, and deliberately so: the one decision that must exclude —
|
|
52
|
+
* claiming a step to execute it — is made by `claimStepForExecution` below as
|
|
53
|
+
* a single conditional write, which needs no lock to be exclusive.
|
|
54
|
+
*
|
|
55
|
+
* What is left are the suspend and approval sections in the engine, where the
|
|
56
|
+
* lock is genuine mutual exclusion rather than a claim. A dialect with a real
|
|
57
|
+
* primitive overrides this to cover them — `kysely-postgres` with
|
|
58
|
+
* `pg_advisory_xact_lock`, `kysely-mysql` with `GET_LOCK`.
|
|
59
|
+
*/
|
|
50
60
|
withStepLock<T>(_runId: string, _stepName: string, fn: () => Promise<T>): Promise<T>;
|
|
61
|
+
/**
|
|
62
|
+
* Claim the step with a status-guarded `UPDATE` and read the affected-row
|
|
63
|
+
* count: the database decides the winner in one statement, so two dispatches
|
|
64
|
+
* racing for the same step cannot both proceed.
|
|
65
|
+
*
|
|
66
|
+
* This replaces the read-then-write the base engine does under `withStepLock`,
|
|
67
|
+
* which is only as exclusive as that lock — and every dialect but Postgres and
|
|
68
|
+
* MySQL inherits a pass-through. A conditional update needs no advisory-lock
|
|
69
|
+
* primitive, so every SQL dialect gets the same guarantee.
|
|
70
|
+
*
|
|
71
|
+
* The winner then goes through the ordinary transition methods, so history
|
|
72
|
+
* rows and the mirror see exactly what they saw before.
|
|
73
|
+
*/
|
|
74
|
+
protected claimStepForExecution(runId: string, stepName: string, rpcName: string): Promise<StepState | null>;
|
|
75
|
+
/**
|
|
76
|
+
* Move a step to `running` only if it is still in one of `from`, reporting
|
|
77
|
+
* whether this caller is the one that moved it.
|
|
78
|
+
*/
|
|
79
|
+
private claimStepStatus;
|
|
51
80
|
getCompletedGraphState(runId: string): Promise<{
|
|
52
81
|
completedNodeIds: string[];
|
|
53
82
|
failedNodeIds: string[];
|
|
@@ -81,16 +110,18 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
81
110
|
private jsonPathFor;
|
|
82
111
|
protected updateRunStateImpl(runId: string, name: string, value: unknown): Promise<void>;
|
|
83
112
|
/**
|
|
84
|
-
* The undispatched-step query, shared by
|
|
113
|
+
* The undispatched-step query, shared by every dialect.
|
|
85
114
|
*
|
|
86
|
-
*
|
|
87
|
-
* `withStepLock` is a pass-through
|
|
88
|
-
*
|
|
89
|
-
* double executions
|
|
90
|
-
*
|
|
91
|
-
*
|
|
115
|
+
* It used to be deliberately NOT an override, because this class's
|
|
116
|
+
* `withStepLock` is a pass-through: a subclass inheriting it (kysely-sqlite)
|
|
117
|
+
* had no atomic claim, so the relay's redundant dispatches would have become
|
|
118
|
+
* double executions, and only `kysely-postgres` and `kysely-mysql` opted in
|
|
119
|
+
* on the strength of their real locks. `claimStepForExecution` no longer
|
|
120
|
+
* needs that lock — the claim is a status-guarded `UPDATE`, atomic in every
|
|
121
|
+
* dialect — so the relay is safe here for all of them and the override lives
|
|
122
|
+
* in the base.
|
|
92
123
|
*/
|
|
93
|
-
protected
|
|
124
|
+
protected findUndispatchedSteps(before: Date, limit: number): Promise<Array<{
|
|
94
125
|
runId: string;
|
|
95
126
|
stepId: string;
|
|
96
127
|
}>>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { PikkuWorkflowService } from '@pikku/core/workflow';
|
|
1
|
+
import { PikkuWorkflowService, WorkflowStepFunctionMismatchError, } from '@pikku/core/workflow';
|
|
2
2
|
import { sql } from 'kysely';
|
|
3
3
|
import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js';
|
|
4
4
|
import { parseJson } from './kysely-json.js';
|
|
5
|
-
import {
|
|
5
|
+
import { requirePikkuSchema } from './schema/index.js';
|
|
6
6
|
import { workflowSchema } from './schema/workflow.schema.js';
|
|
7
7
|
export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
8
8
|
db;
|
|
@@ -16,7 +16,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
16
16
|
async init() {
|
|
17
17
|
if (this.initialized)
|
|
18
18
|
return;
|
|
19
|
-
await
|
|
19
|
+
await requirePikkuSchema(this.db, workflowSchema);
|
|
20
20
|
this.initialized = true;
|
|
21
21
|
}
|
|
22
22
|
async createRunImpl(workflowName, input, inline, graphHash, wire, options) {
|
|
@@ -363,9 +363,73 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
363
363
|
async withRunLock(_id, fn) {
|
|
364
364
|
return fn();
|
|
365
365
|
}
|
|
366
|
+
/**
|
|
367
|
+
* A pass-through, and deliberately so: the one decision that must exclude —
|
|
368
|
+
* claiming a step to execute it — is made by `claimStepForExecution` below as
|
|
369
|
+
* a single conditional write, which needs no lock to be exclusive.
|
|
370
|
+
*
|
|
371
|
+
* What is left are the suspend and approval sections in the engine, where the
|
|
372
|
+
* lock is genuine mutual exclusion rather than a claim. A dialect with a real
|
|
373
|
+
* primitive overrides this to cover them — `kysely-postgres` with
|
|
374
|
+
* `pg_advisory_xact_lock`, `kysely-mysql` with `GET_LOCK`.
|
|
375
|
+
*/
|
|
366
376
|
async withStepLock(_runId, _stepName, fn) {
|
|
367
377
|
return fn();
|
|
368
378
|
}
|
|
379
|
+
/**
|
|
380
|
+
* Claim the step with a status-guarded `UPDATE` and read the affected-row
|
|
381
|
+
* count: the database decides the winner in one statement, so two dispatches
|
|
382
|
+
* racing for the same step cannot both proceed.
|
|
383
|
+
*
|
|
384
|
+
* This replaces the read-then-write the base engine does under `withStepLock`,
|
|
385
|
+
* which is only as exclusive as that lock — and every dialect but Postgres and
|
|
386
|
+
* MySQL inherits a pass-through. A conditional update needs no advisory-lock
|
|
387
|
+
* primitive, so every SQL dialect gets the same guarantee.
|
|
388
|
+
*
|
|
389
|
+
* The winner then goes through the ordinary transition methods, so history
|
|
390
|
+
* rows and the mirror see exactly what they saw before.
|
|
391
|
+
*/
|
|
392
|
+
async claimStepForExecution(runId, stepName, rpcName) {
|
|
393
|
+
const stepState = await this.getStepState(runId, stepName);
|
|
394
|
+
// knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
|
|
395
|
+
if (stepState.rpcName !== undefined &&
|
|
396
|
+
stepState.rpcName !== (rpcName ?? null)) {
|
|
397
|
+
throw new WorkflowStepFunctionMismatchError(runId, stepName);
|
|
398
|
+
}
|
|
399
|
+
if (stepState.status === 'succeeded' || stepState.status === 'running') {
|
|
400
|
+
return null;
|
|
401
|
+
}
|
|
402
|
+
if (stepState.status === 'failed') {
|
|
403
|
+
if (!(await this.claimStepStatus(stepState.stepId, ['failed']))) {
|
|
404
|
+
return null;
|
|
405
|
+
}
|
|
406
|
+
return this.createRetryAttempt(stepState.stepId, 'running');
|
|
407
|
+
}
|
|
408
|
+
if (stepState.status === 'pending' || stepState.status === 'scheduled') {
|
|
409
|
+
if (!(await this.claimStepStatus(stepState.stepId, [
|
|
410
|
+
'pending',
|
|
411
|
+
'scheduled',
|
|
412
|
+
]))) {
|
|
413
|
+
return null;
|
|
414
|
+
}
|
|
415
|
+
await this.setStepRunning(stepState.stepId);
|
|
416
|
+
return { ...stepState, status: 'running' };
|
|
417
|
+
}
|
|
418
|
+
return stepState;
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* Move a step to `running` only if it is still in one of `from`, reporting
|
|
422
|
+
* whether this caller is the one that moved it.
|
|
423
|
+
*/
|
|
424
|
+
async claimStepStatus(stepId, from) {
|
|
425
|
+
const claimed = await this.db
|
|
426
|
+
.updateTable('workflowStep')
|
|
427
|
+
.set({ status: 'running', updatedAt: new Date() })
|
|
428
|
+
.where('workflowStepId', '=', stepId)
|
|
429
|
+
.where('status', 'in', from)
|
|
430
|
+
.executeTakeFirst();
|
|
431
|
+
return Number(claimed?.numUpdatedRows ?? 0n) > 0;
|
|
432
|
+
}
|
|
369
433
|
async getCompletedGraphState(runId) {
|
|
370
434
|
const results = await this.db
|
|
371
435
|
.selectFrom('workflowStep as ws')
|
|
@@ -470,16 +534,18 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
470
534
|
.execute();
|
|
471
535
|
}
|
|
472
536
|
/**
|
|
473
|
-
* The undispatched-step query, shared by
|
|
537
|
+
* The undispatched-step query, shared by every dialect.
|
|
474
538
|
*
|
|
475
|
-
*
|
|
476
|
-
* `withStepLock` is a pass-through
|
|
477
|
-
*
|
|
478
|
-
* double executions
|
|
479
|
-
*
|
|
480
|
-
*
|
|
539
|
+
* It used to be deliberately NOT an override, because this class's
|
|
540
|
+
* `withStepLock` is a pass-through: a subclass inheriting it (kysely-sqlite)
|
|
541
|
+
* had no atomic claim, so the relay's redundant dispatches would have become
|
|
542
|
+
* double executions, and only `kysely-postgres` and `kysely-mysql` opted in
|
|
543
|
+
* on the strength of their real locks. `claimStepForExecution` no longer
|
|
544
|
+
* needs that lock — the claim is a status-guarded `UPDATE`, atomic in every
|
|
545
|
+
* dialect — so the relay is safe here for all of them and the override lives
|
|
546
|
+
* in the base.
|
|
481
547
|
*/
|
|
482
|
-
async
|
|
548
|
+
async findUndispatchedSteps(before, limit) {
|
|
483
549
|
const rows = await this.db
|
|
484
550
|
.selectFrom('workflowStep as s')
|
|
485
551
|
.innerJoin('workflowRuns as r', 'r.workflowRunId', 's.workflowRunId')
|
|
@@ -2,10 +2,11 @@ import type { PikkuSchema } from './pikku-schema.types.js';
|
|
|
2
2
|
/**
|
|
3
3
|
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* Gated by `ownedBy` rather than written for everyone: only a project that has
|
|
6
|
+
* wired a durable audit sink needs this table, and generating it for the rest
|
|
7
|
+
* would put an empty table in every database. `KyselyAuditService.init()`
|
|
8
|
+
* requires it and never creates it, so it arrives with the migration `pikku db
|
|
9
|
+
* generate` writes for the sink that fills it.
|
|
9
10
|
*
|
|
10
11
|
* Every column is text on every engine, matching the platform audit-queue
|
|
11
12
|
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Gated by `ownedBy` rather than written for everyone: only a project that has
|
|
5
|
+
* wired a durable audit sink needs this table, and generating it for the rest
|
|
6
|
+
* would put an empty table in every database. `KyselyAuditService.init()`
|
|
7
|
+
* requires it and never creates it, so it arrives with the migration `pikku db
|
|
8
|
+
* generate` writes for the sink that fills it.
|
|
8
9
|
*
|
|
9
10
|
* Every column is text on every engine, matching the platform audit-queue
|
|
10
11
|
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
@@ -14,6 +15,7 @@
|
|
|
14
15
|
*/
|
|
15
16
|
export const auditSchema = {
|
|
16
17
|
name: 'audit',
|
|
18
|
+
ownedBy: ['audit', 'auditLog'],
|
|
17
19
|
statements: [
|
|
18
20
|
(db) => db.schema
|
|
19
21
|
.createTable('audit')
|