@pikku/kysely 0.13.6 → 0.13.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +505 -0
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +3 -1
- package/dist/src/kysely-audit-service.d.ts +34 -2
- package/dist/src/kysely-audit-service.js +163 -3
- package/dist/src/kysely-credential-service.d.ts +3 -0
- package/dist/src/kysely-credential-service.js +50 -11
- package/dist/src/kysely-scope-service.d.ts +38 -0
- package/dist/src/kysely-scope-service.js +125 -2
- package/dist/src/kysely-secret-service.d.ts +7 -3
- package/dist/src/kysely-secret-service.js +55 -13
- package/dist/src/kysely-tables.d.ts +49 -2
- package/dist/src/kysely-virtual-user-run-store.d.ts +43 -0
- package/dist/src/kysely-virtual-user-run-store.js +131 -0
- package/dist/src/kysely-workflow-service.d.ts +0 -1
- package/dist/src/kysely-workflow-service.js +0 -12
- package/dist/src/schema/audit.schema.d.ts +16 -0
- package/dist/src/schema/audit.schema.js +47 -0
- package/dist/src/schema/credential.schema.d.ts +3 -2
- package/dist/src/schema/credential.schema.js +9 -3
- package/dist/src/schema/index.d.ts +17 -4
- package/dist/src/schema/index.js +40 -6
- package/dist/src/schema/pikku-schema.types.d.ts +41 -5
- package/dist/src/schema/pikku-schema.types.js +27 -12
- package/dist/src/schema/scope.schema.js +12 -0
- package/dist/src/schema/secret.schema.js +5 -0
- package/dist/src/schema/virtual-user.schema.d.ts +19 -0
- package/dist/src/schema/virtual-user.schema.js +54 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/create-audited-kysely.test.ts +10 -10
- package/src/index.ts +7 -0
- package/src/kysely-audit-service.test.ts +360 -0
- package/src/kysely-audit-service.ts +181 -4
- package/src/kysely-credential-service.ts +57 -12
- package/src/kysely-scope-service.ts +151 -2
- package/src/kysely-secret-service.ts +70 -21
- package/src/kysely-services.test.ts +123 -1
- package/src/kysely-tables.test.ts +12 -1
- package/src/kysely-tables.ts +53 -2
- package/src/kysely-virtual-user-run-store.test.ts +212 -0
- package/src/kysely-virtual-user-run-store.ts +149 -0
- package/src/kysely-workflow-service.ts +0 -17
- package/src/schema/audit.schema.ts +55 -0
- package/src/schema/credential.schema.ts +14 -3
- package/src/schema/index.ts +59 -10
- package/src/schema/pikku-schema.types.ts +65 -10
- package/src/schema/schema.test.ts +82 -4
- package/src/schema/scope.schema.ts +14 -0
- package/src/schema/secret.schema.ts +9 -0
- package/src/schema/virtual-user.schema.ts +65 -0
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createSecretValue, isSecretValue } from '@pikku/core';
|
|
2
|
+
import { deriveKEK, generateKEKSalt, envelopeEncrypt, envelopeDecrypt, envelopeRewrap, } from '@pikku/core/crypto-utils';
|
|
2
3
|
import { ensurePikkuSchema } from './schema/index.js';
|
|
3
4
|
import { secretSchema } from './schema/secret.schema.js';
|
|
4
5
|
export class KyselySecretService {
|
|
5
6
|
db;
|
|
6
7
|
initialized = false;
|
|
8
|
+
kekSalts = new Map();
|
|
9
|
+
keks = new Map();
|
|
7
10
|
key;
|
|
8
11
|
keyVersion;
|
|
9
12
|
previousKey;
|
|
@@ -38,12 +41,47 @@ export class KyselySecretService {
|
|
|
38
41
|
})
|
|
39
42
|
.execute();
|
|
40
43
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
async getKEKSalt(version) {
|
|
45
|
+
const cached = this.kekSalts.get(version);
|
|
46
|
+
if (cached)
|
|
47
|
+
return cached;
|
|
48
|
+
const existing = await this.db
|
|
49
|
+
.selectFrom('secretKekSalts')
|
|
50
|
+
.select('salt')
|
|
51
|
+
.where('keyVersion', '=', version)
|
|
52
|
+
.executeTakeFirst();
|
|
53
|
+
let salt = existing?.salt;
|
|
54
|
+
if (!salt) {
|
|
55
|
+
await this.db
|
|
56
|
+
.insertInto('secretKekSalts')
|
|
57
|
+
.values({
|
|
58
|
+
keyVersion: version,
|
|
59
|
+
salt: generateKEKSalt(),
|
|
60
|
+
createdAt: new Date().toISOString(),
|
|
61
|
+
})
|
|
62
|
+
.onConflict((oc) => oc.column('keyVersion').doNothing())
|
|
63
|
+
.execute();
|
|
64
|
+
const row = await this.db
|
|
65
|
+
.selectFrom('secretKekSalts')
|
|
66
|
+
.select('salt')
|
|
67
|
+
.where('keyVersion', '=', version)
|
|
68
|
+
.executeTakeFirstOrThrow();
|
|
69
|
+
salt = row.salt;
|
|
70
|
+
}
|
|
71
|
+
this.kekSalts.set(version, salt);
|
|
72
|
+
return salt;
|
|
73
|
+
}
|
|
74
|
+
async getKEK(version) {
|
|
75
|
+
const cached = this.keks.get(version);
|
|
76
|
+
if (cached)
|
|
77
|
+
return cached;
|
|
78
|
+
const passphrase = version === this.keyVersion ? this.key : this.previousKey;
|
|
79
|
+
if (!passphrase) {
|
|
80
|
+
throw new Error(`No KEK available for key_version ${version}`);
|
|
81
|
+
}
|
|
82
|
+
const kek = await deriveKEK(passphrase, await this.getKEKSalt(version));
|
|
83
|
+
this.keks.set(version, kek);
|
|
84
|
+
return kek;
|
|
47
85
|
}
|
|
48
86
|
async getSecret(key) {
|
|
49
87
|
const row = await this.db
|
|
@@ -53,10 +91,10 @@ export class KyselySecretService {
|
|
|
53
91
|
.executeTakeFirst();
|
|
54
92
|
if (!row)
|
|
55
93
|
throw new Error('Requested secret not found');
|
|
56
|
-
const kek = this.getKEK(row.keyVersion);
|
|
94
|
+
const kek = await this.getKEK(row.keyVersion);
|
|
57
95
|
const result = await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek);
|
|
58
96
|
await this.logAudit(key, 'read');
|
|
59
|
-
return result;
|
|
97
|
+
return createSecretValue(result);
|
|
60
98
|
}
|
|
61
99
|
async hasSecret(key) {
|
|
62
100
|
const row = await this.db
|
|
@@ -67,7 +105,9 @@ export class KyselySecretService {
|
|
|
67
105
|
return !!row;
|
|
68
106
|
}
|
|
69
107
|
async setSecret(key, value) {
|
|
70
|
-
|
|
108
|
+
// Encrypting the wrapper would store its redaction, not the secret —
|
|
109
|
+
// writing a secret back to the vault is exactly what this method is for.
|
|
110
|
+
const { ciphertext, wrappedDEK } = await envelopeEncrypt(await this.getKEK(this.keyVersion), isSecretValue(value) ? value.reveal() : value);
|
|
71
111
|
const now = new Date().toISOString();
|
|
72
112
|
await this.db
|
|
73
113
|
.insertInto('secrets')
|
|
@@ -101,8 +141,8 @@ export class KyselySecretService {
|
|
|
101
141
|
const out = {};
|
|
102
142
|
for (const row of rows) {
|
|
103
143
|
try {
|
|
104
|
-
const kek = this.getKEK(row.keyVersion);
|
|
105
|
-
out[row.key] = await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek);
|
|
144
|
+
const kek = await this.getKEK(row.keyVersion);
|
|
145
|
+
out[row.key] = createSecretValue(await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek));
|
|
106
146
|
}
|
|
107
147
|
catch (cause) {
|
|
108
148
|
// A stored secret that fails to decrypt is never expected — it means the
|
|
@@ -126,8 +166,10 @@ export class KyselySecretService {
|
|
|
126
166
|
.select(['key', 'wrappedDek'])
|
|
127
167
|
.where('keyVersion', '<', this.keyVersion)
|
|
128
168
|
.execute();
|
|
169
|
+
const oldKEK = await this.getKEK(this.keyVersion - 1);
|
|
170
|
+
const newKEK = await this.getKEK(this.keyVersion);
|
|
129
171
|
for (const row of rows) {
|
|
130
|
-
const newWrappedDEK = await envelopeRewrap(
|
|
172
|
+
const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek);
|
|
131
173
|
await this.db
|
|
132
174
|
.updateTable('secrets')
|
|
133
175
|
.set({
|
|
@@ -116,7 +116,12 @@ export interface AIRunTable {
|
|
|
116
116
|
agentName: string;
|
|
117
117
|
threadId: string;
|
|
118
118
|
resourceId: string;
|
|
119
|
-
|
|
119
|
+
/**
|
|
120
|
+
* `interrupted` is distinct from `failed`: the run was stopped on purpose,
|
|
121
|
+
* usually because the user talked over it. Nothing went wrong, so it should
|
|
122
|
+
* not show up in error reporting.
|
|
123
|
+
*/
|
|
124
|
+
status: Generated<'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'>;
|
|
120
125
|
errorMessage: string | null;
|
|
121
126
|
suspendReason: 'approval' | 'credential' | 'rpc-missing' | null;
|
|
122
127
|
missingRpcs: string | null;
|
|
@@ -146,6 +151,11 @@ export interface SecretsTable {
|
|
|
146
151
|
createdAt: Generated<Date>;
|
|
147
152
|
updatedAt: Generated<Date>;
|
|
148
153
|
}
|
|
154
|
+
export interface KekSaltsTable {
|
|
155
|
+
keyVersion: number;
|
|
156
|
+
salt: string;
|
|
157
|
+
createdAt: Generated<Date>;
|
|
158
|
+
}
|
|
149
159
|
export interface SecretsAuditTable {
|
|
150
160
|
id: string;
|
|
151
161
|
secretKey: string;
|
|
@@ -191,10 +201,14 @@ export interface PikkuScopesTable {
|
|
|
191
201
|
*/
|
|
192
202
|
declared: Generated<boolean>;
|
|
193
203
|
}
|
|
194
|
-
/** A role: an admin
|
|
204
|
+
/** A role: a bag of scopes, composed by an admin or declared in code. */
|
|
195
205
|
export interface PikkuRolesTable {
|
|
196
206
|
name: string;
|
|
197
207
|
description: string | null;
|
|
208
|
+
/** Declared with `defineSystemRole`, and therefore not editable here. */
|
|
209
|
+
system: Generated<boolean>;
|
|
210
|
+
/** False for a system role whose declaration has gone: held, but inert. */
|
|
211
|
+
declared: Generated<boolean>;
|
|
198
212
|
createdAt: Generated<Date>;
|
|
199
213
|
}
|
|
200
214
|
export interface PikkuRoleScopesTable {
|
|
@@ -234,6 +248,36 @@ export interface WebhookDeliveryAttemptTable {
|
|
|
234
248
|
error: string | null;
|
|
235
249
|
createdAt: Generated<Date>;
|
|
236
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* One virtual-user run. The JSON columns (`goals`, `memory`, `findings`,
|
|
253
|
+
* `tally`) are text the store serialises itself, so the row is byte-identical
|
|
254
|
+
* on every engine.
|
|
255
|
+
*/
|
|
256
|
+
export interface VirtualUserRunTable {
|
|
257
|
+
runId: string;
|
|
258
|
+
persona: string;
|
|
259
|
+
disposition: string;
|
|
260
|
+
/**
|
|
261
|
+
* Read back as a string by some drivers — a BIGINT does not always fit a JS
|
|
262
|
+
* number — so the store narrows it rather than trusting the column type.
|
|
263
|
+
*/
|
|
264
|
+
seed: string | number | bigint;
|
|
265
|
+
status: Generated<'running' | 'completed' | 'failed'>;
|
|
266
|
+
goals: Generated<string>;
|
|
267
|
+
memory: Generated<string>;
|
|
268
|
+
findings: Generated<string>;
|
|
269
|
+
tally: string | null;
|
|
270
|
+
stoppedBy: string | null;
|
|
271
|
+
error: string | null;
|
|
272
|
+
startedBy: string | null;
|
|
273
|
+
/**
|
|
274
|
+
* Written as an ISO string and read back as whatever the driver returns — a
|
|
275
|
+
* bare SQLite driver cannot bind a `Date` at all, and the store normalises
|
|
276
|
+
* both ends.
|
|
277
|
+
*/
|
|
278
|
+
createdAt: Generated<Date | string>;
|
|
279
|
+
finishedAt: Date | string | null;
|
|
280
|
+
}
|
|
237
281
|
export interface KyselyPikkuDB {
|
|
238
282
|
pikkuScopes: PikkuScopesTable;
|
|
239
283
|
pikkuRoles: PikkuRolesTable;
|
|
@@ -254,10 +298,13 @@ export interface KyselyPikkuDB {
|
|
|
254
298
|
pikkuDeployments: PikkuDeploymentsTable;
|
|
255
299
|
pikkuDeploymentFunctions: PikkuDeploymentFunctionsTable;
|
|
256
300
|
secrets: SecretsTable;
|
|
301
|
+
secretKekSalts: KekSaltsTable;
|
|
257
302
|
secretsAudit: SecretsAuditTable;
|
|
258
303
|
credentials: CredentialsTable;
|
|
304
|
+
credentialKekSalts: KekSaltsTable;
|
|
259
305
|
credentialsAudit: CredentialsAuditTable;
|
|
260
306
|
pikkuUserSessions: UserSessionsTable;
|
|
261
307
|
webhookDelivery: WebhookDeliveryTable;
|
|
262
308
|
webhookDeliveryAttempt: WebhookDeliveryAttemptTable;
|
|
309
|
+
virtualUserRun: VirtualUserRunTable;
|
|
263
310
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Kysely } from 'kysely';
|
|
2
|
+
import type { VirtualUserRunOutcome, VirtualUserRunRecord, VirtualUserRunStart, VirtualUserRunStore } from '@pikku/core/virtual-user';
|
|
3
|
+
import type { KyselyPikkuDB } from './kysely-tables.js';
|
|
4
|
+
/**
|
|
5
|
+
* Records virtual-user runs in a `virtualUserRun` table.
|
|
6
|
+
*
|
|
7
|
+
* The store is the whole record of a run: `runVirtualUser` returns as soon as
|
|
8
|
+
* the row exists and the engine keeps going without it, so nothing else knows
|
|
9
|
+
* the run happened. That is why `fail()` exists as its own call — a run that
|
|
10
|
+
* crashed and a run that found nothing are different answers, and a row left at
|
|
11
|
+
* `running` is neither.
|
|
12
|
+
*
|
|
13
|
+
* Works with or without `SerializePlugin`, which not every project installs: the
|
|
14
|
+
* JSON columns are written as strings (which the plugin passes through) and read
|
|
15
|
+
* back through `parseJson` (which passes through what the plugin already
|
|
16
|
+
* parsed). Timestamps are written as ISO strings for the same reason — a bare
|
|
17
|
+
* SQLite driver cannot bind a `Date` at all.
|
|
18
|
+
*
|
|
19
|
+
* SECURITY: an `adversarial` run's findings are working exploits carrying live
|
|
20
|
+
* ids. This is a privileged store; the scaffolded reads are scope-gated, and
|
|
21
|
+
* exposing these rows more widely publishes your own exploits.
|
|
22
|
+
*/
|
|
23
|
+
export declare class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
24
|
+
private db;
|
|
25
|
+
private initialized;
|
|
26
|
+
constructor(db: Kysely<KyselyPikkuDB>);
|
|
27
|
+
/**
|
|
28
|
+
* Creates the table on first use, like the audit sink — the runtime does not
|
|
29
|
+
* need it, so it arrives with the feature that fills it rather than in every
|
|
30
|
+
* database.
|
|
31
|
+
*/
|
|
32
|
+
init(): Promise<void>;
|
|
33
|
+
start(run: VirtualUserRunStart): Promise<string>;
|
|
34
|
+
complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void>;
|
|
35
|
+
fail(runId: string, error: string): Promise<void>;
|
|
36
|
+
get(runId: string): Promise<VirtualUserRunRecord | null>;
|
|
37
|
+
list(options?: {
|
|
38
|
+
persona?: string;
|
|
39
|
+
limit?: number;
|
|
40
|
+
offset?: number;
|
|
41
|
+
}): Promise<VirtualUserRunRecord[]>;
|
|
42
|
+
private toRecord;
|
|
43
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { parseJson } from './kysely-json.js';
|
|
3
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
4
|
+
import { virtualUserSchema } from './schema/virtual-user.schema.js';
|
|
5
|
+
/**
|
|
6
|
+
* Records virtual-user runs in a `virtualUserRun` table.
|
|
7
|
+
*
|
|
8
|
+
* The store is the whole record of a run: `runVirtualUser` returns as soon as
|
|
9
|
+
* the row exists and the engine keeps going without it, so nothing else knows
|
|
10
|
+
* the run happened. That is why `fail()` exists as its own call — a run that
|
|
11
|
+
* crashed and a run that found nothing are different answers, and a row left at
|
|
12
|
+
* `running` is neither.
|
|
13
|
+
*
|
|
14
|
+
* Works with or without `SerializePlugin`, which not every project installs: the
|
|
15
|
+
* JSON columns are written as strings (which the plugin passes through) and read
|
|
16
|
+
* back through `parseJson` (which passes through what the plugin already
|
|
17
|
+
* parsed). Timestamps are written as ISO strings for the same reason — a bare
|
|
18
|
+
* SQLite driver cannot bind a `Date` at all.
|
|
19
|
+
*
|
|
20
|
+
* SECURITY: an `adversarial` run's findings are working exploits carrying live
|
|
21
|
+
* ids. This is a privileged store; the scaffolded reads are scope-gated, and
|
|
22
|
+
* exposing these rows more widely publishes your own exploits.
|
|
23
|
+
*/
|
|
24
|
+
export class KyselyVirtualUserRunStore {
|
|
25
|
+
db;
|
|
26
|
+
initialized = false;
|
|
27
|
+
constructor(db) {
|
|
28
|
+
this.db = db;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Creates the table on first use, like the audit sink — the runtime does not
|
|
32
|
+
* need it, so it arrives with the feature that fills it rather than in every
|
|
33
|
+
* database.
|
|
34
|
+
*/
|
|
35
|
+
async init() {
|
|
36
|
+
if (this.initialized)
|
|
37
|
+
return;
|
|
38
|
+
await ensurePikkuSchema(this.db, virtualUserSchema);
|
|
39
|
+
this.initialized = true;
|
|
40
|
+
}
|
|
41
|
+
async start(run) {
|
|
42
|
+
await this.init();
|
|
43
|
+
const runId = randomUUID();
|
|
44
|
+
await this.db
|
|
45
|
+
.insertInto('virtualUserRun')
|
|
46
|
+
.values({
|
|
47
|
+
runId,
|
|
48
|
+
persona: run.persona,
|
|
49
|
+
disposition: run.disposition,
|
|
50
|
+
// Text on every engine (see the schema), so the seed round-trips
|
|
51
|
+
// through drivers that hand a BIGINT back as a string.
|
|
52
|
+
seed: String(run.seed),
|
|
53
|
+
status: 'running',
|
|
54
|
+
goals: JSON.stringify(run.goals ?? []),
|
|
55
|
+
memory: JSON.stringify(run.memory ?? {}),
|
|
56
|
+
findings: '[]',
|
|
57
|
+
startedBy: run.startedBy ?? null,
|
|
58
|
+
})
|
|
59
|
+
.execute();
|
|
60
|
+
return runId;
|
|
61
|
+
}
|
|
62
|
+
async complete(runId, outcome) {
|
|
63
|
+
await this.init();
|
|
64
|
+
await this.db
|
|
65
|
+
.updateTable('virtualUserRun')
|
|
66
|
+
.set({
|
|
67
|
+
status: 'completed',
|
|
68
|
+
findings: JSON.stringify(outcome.findings),
|
|
69
|
+
tally: JSON.stringify(outcome.tally),
|
|
70
|
+
// Overwritten rather than merged: the engine's memory already carries
|
|
71
|
+
// what it was given, plus what it learned on the way.
|
|
72
|
+
memory: JSON.stringify(outcome.memory),
|
|
73
|
+
stoppedBy: outcome.stoppedBy,
|
|
74
|
+
finishedAt: new Date().toISOString(),
|
|
75
|
+
})
|
|
76
|
+
.where('runId', '=', runId)
|
|
77
|
+
.execute();
|
|
78
|
+
}
|
|
79
|
+
async fail(runId, error) {
|
|
80
|
+
await this.init();
|
|
81
|
+
await this.db
|
|
82
|
+
.updateTable('virtualUserRun')
|
|
83
|
+
.set({
|
|
84
|
+
status: 'failed',
|
|
85
|
+
error,
|
|
86
|
+
finishedAt: new Date().toISOString(),
|
|
87
|
+
})
|
|
88
|
+
.where('runId', '=', runId)
|
|
89
|
+
.execute();
|
|
90
|
+
}
|
|
91
|
+
async get(runId) {
|
|
92
|
+
await this.init();
|
|
93
|
+
const row = await this.db
|
|
94
|
+
.selectFrom('virtualUserRun')
|
|
95
|
+
.selectAll()
|
|
96
|
+
.where('runId', '=', runId)
|
|
97
|
+
.executeTakeFirst();
|
|
98
|
+
return row ? this.toRecord(row) : null;
|
|
99
|
+
}
|
|
100
|
+
async list(options) {
|
|
101
|
+
await this.init();
|
|
102
|
+
let query = this.db.selectFrom('virtualUserRun').selectAll();
|
|
103
|
+
if (options?.persona) {
|
|
104
|
+
query = query.where('persona', '=', options.persona);
|
|
105
|
+
}
|
|
106
|
+
const rows = await query
|
|
107
|
+
.orderBy('createdAt', 'desc')
|
|
108
|
+
.limit(options?.limit ?? 50)
|
|
109
|
+
.offset(options?.offset ?? 0)
|
|
110
|
+
.execute();
|
|
111
|
+
return rows.map((row) => this.toRecord(row));
|
|
112
|
+
}
|
|
113
|
+
toRecord(row) {
|
|
114
|
+
return {
|
|
115
|
+
runId: row.runId,
|
|
116
|
+
persona: row.persona,
|
|
117
|
+
disposition: row.disposition,
|
|
118
|
+
seed: Number(row.seed),
|
|
119
|
+
status: row.status,
|
|
120
|
+
goals: parseJson(row.goals) ?? [],
|
|
121
|
+
memory: parseJson(row.memory) ?? {},
|
|
122
|
+
findings: parseJson(row.findings) ?? [],
|
|
123
|
+
tally: row.tally ? (parseJson(row.tally) ?? null) : null,
|
|
124
|
+
stoppedBy: row.stoppedBy,
|
|
125
|
+
error: row.error,
|
|
126
|
+
startedBy: row.startedBy,
|
|
127
|
+
createdAt: new Date(row.createdAt),
|
|
128
|
+
finishedAt: row.finishedAt ? new Date(row.finishedAt) : null,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -52,7 +52,6 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
52
52
|
failedNodeIds: string[];
|
|
53
53
|
branchKeys: Record<string, string>;
|
|
54
54
|
}>;
|
|
55
|
-
getNodesWithoutSteps(runId: string, nodeIds: string[]): Promise<string[]>;
|
|
56
55
|
getStepInstances(runId: string): Promise<Array<{
|
|
57
56
|
stepName: string;
|
|
58
57
|
status: StepStatus;
|
|
@@ -393,18 +393,6 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
393
393
|
}
|
|
394
394
|
return { completedNodeIds, failedNodeIds, branchKeys };
|
|
395
395
|
}
|
|
396
|
-
async getNodesWithoutSteps(runId, nodeIds) {
|
|
397
|
-
if (nodeIds.length === 0)
|
|
398
|
-
return [];
|
|
399
|
-
const result = await this.db
|
|
400
|
-
.selectFrom('workflowStep')
|
|
401
|
-
.select('stepName')
|
|
402
|
-
.where('workflowRunId', '=', runId)
|
|
403
|
-
.where('stepName', 'in', nodeIds)
|
|
404
|
-
.execute();
|
|
405
|
-
const existingStepNames = new Set(result.map((r) => r.stepName));
|
|
406
|
-
return nodeIds.filter((id) => !existingStepNames.has(id));
|
|
407
|
-
}
|
|
408
396
|
async getStepInstances(runId) {
|
|
409
397
|
const rows = await this.db
|
|
410
398
|
.selectFrom('workflowStep')
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { PikkuSchema } from './pikku-schema.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
4
|
+
*
|
|
5
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
6
|
+
* a project that has wired a durable audit sink does, and creating it for
|
|
7
|
+
* everyone would put an empty table in every database. `KyselyAuditService.init()`
|
|
8
|
+
* applies it, so it arrives exactly when the sink that fills it does.
|
|
9
|
+
*
|
|
10
|
+
* Every column is text on every engine, matching the platform audit-queue
|
|
11
|
+
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
12
|
+
* the same reader can read. `occurredAt` is an ISO 8601 string rather than a
|
|
13
|
+
* timestamp for that reason: string ordering is chronological ordering, and the
|
|
14
|
+
* queue consumer has no shared type to agree on.
|
|
15
|
+
*/
|
|
16
|
+
export declare const auditSchema: PikkuSchema;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
5
|
+
* a project that has wired a durable audit sink does, and creating it for
|
|
6
|
+
* everyone would put an empty table in every database. `KyselyAuditService.init()`
|
|
7
|
+
* applies it, so it arrives exactly when the sink that fills it does.
|
|
8
|
+
*
|
|
9
|
+
* Every column is text on every engine, matching the platform audit-queue
|
|
10
|
+
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
11
|
+
* the same reader can read. `occurredAt` is an ISO 8601 string rather than a
|
|
12
|
+
* timestamp for that reason: string ordering is chronological ordering, and the
|
|
13
|
+
* queue consumer has no shared type to agree on.
|
|
14
|
+
*/
|
|
15
|
+
export const auditSchema = {
|
|
16
|
+
name: 'audit',
|
|
17
|
+
statements: [
|
|
18
|
+
(db) => db.schema
|
|
19
|
+
.createTable('audit')
|
|
20
|
+
.addColumn('auditId', 'text', (col) => col.primaryKey())
|
|
21
|
+
.addColumn('occurredAt', 'text', (col) => col.notNull())
|
|
22
|
+
.addColumn('type', 'text', (col) => col.notNull())
|
|
23
|
+
.addColumn('source', 'text', (col) => col.defaultTo('auto').notNull())
|
|
24
|
+
.addColumn('outcome', 'text')
|
|
25
|
+
.addColumn('functionId', 'text')
|
|
26
|
+
.addColumn('wireType', 'text')
|
|
27
|
+
.addColumn('traceId', 'text')
|
|
28
|
+
.addColumn('transactionId', 'text')
|
|
29
|
+
.addColumn('queryId', 'text')
|
|
30
|
+
.addColumn('userId', 'text')
|
|
31
|
+
.addColumn('orgId', 'text')
|
|
32
|
+
.addColumn('pikkuUserId', 'text')
|
|
33
|
+
.addColumn('tables', 'text')
|
|
34
|
+
.addColumn('changedCols', 'text')
|
|
35
|
+
.addColumn('event', 'text')
|
|
36
|
+
.addColumn('old', 'text')
|
|
37
|
+
.addColumn('data', 'text'),
|
|
38
|
+
// The trail is only ever read newest-first, and the two filters the console
|
|
39
|
+
// offers are user and type.
|
|
40
|
+
(db) => db.schema
|
|
41
|
+
.createIndex('idx_audit_occurred_at')
|
|
42
|
+
.on('audit')
|
|
43
|
+
.column('occurredAt'),
|
|
44
|
+
(db) => db.schema.createIndex('idx_audit_user_id').on('audit').column('userId'),
|
|
45
|
+
(db) => db.schema.createIndex('idx_audit_type').on('audit').column('type'),
|
|
46
|
+
],
|
|
47
|
+
};
|
|
@@ -5,7 +5,8 @@ import { type PikkuSchema } from './pikku-schema.types.js';
|
|
|
5
5
|
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
6
6
|
* rows with a null user must still collide on name — which the schema builder
|
|
7
7
|
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
8
|
-
* which is why the table
|
|
9
|
-
* path.
|
|
8
|
+
* which is why the table goes through `ctx.table` rather than being left to the
|
|
9
|
+
* search path. The index's own name stays unqualified: an index belongs to the
|
|
10
|
+
* schema of the table it is on, and postgres rejects one that says otherwise.
|
|
10
11
|
*/
|
|
11
12
|
export declare const credentialSchema: PikkuSchema;
|
|
@@ -6,8 +6,9 @@ import { rawStatement } from './pikku-schema.types.js';
|
|
|
6
6
|
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
7
7
|
* rows with a null user must still collide on name — which the schema builder
|
|
8
8
|
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
9
|
-
* which is why the table
|
|
10
|
-
* path.
|
|
9
|
+
* which is why the table goes through `ctx.table` rather than being left to the
|
|
10
|
+
* search path. The index's own name stays unqualified: an index belongs to the
|
|
11
|
+
* schema of the table it is on, and postgres rejects one that says otherwise.
|
|
11
12
|
*/
|
|
12
13
|
export const credentialSchema = {
|
|
13
14
|
name: 'credential',
|
|
@@ -21,7 +22,12 @@ export const credentialSchema = {
|
|
|
21
22
|
.addColumn('keyVersion', 'integer', (col) => col.notNull())
|
|
22
23
|
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
23
24
|
.addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
|
|
24
|
-
rawStatement(sql `CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`),
|
|
25
|
+
rawStatement(({ table }) => sql `CREATE UNIQUE INDEX credentials_name_user_id_unique ON ${table('credentials')} (name, COALESCE(user_id, ''))`),
|
|
26
|
+
(db) => db.schema
|
|
27
|
+
.createTable('credentialKekSalts')
|
|
28
|
+
.addColumn('keyVersion', 'integer', (col) => col.primaryKey())
|
|
29
|
+
.addColumn('salt', 'varchar(64)', (col) => col.notNull())
|
|
30
|
+
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
|
|
25
31
|
(db) => db.schema
|
|
26
32
|
.createTable('credentialsAudit')
|
|
27
33
|
.addColumn('id', 'varchar(36)', (col) => col.primaryKey())
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { type Kysely } from 'kysely';
|
|
2
|
-
import type
|
|
3
|
-
export type { PikkuSchema, RequiredTypes, SchemaRequirement, SchemaStatement, SchemaStatementFactory, } from './pikku-schema.types.js';
|
|
4
|
-
export { rawStatement, requiredType } from './pikku-schema.types.js';
|
|
2
|
+
import { type PikkuSchema, type RequiredTypes, type SchemaRequirement } from './pikku-schema.types.js';
|
|
3
|
+
export type { PikkuSchema, RequiredTypes, SchemaContext, SchemaRequirement, SchemaStatement, SchemaStatementFactory, } from './pikku-schema.types.js';
|
|
4
|
+
export { rawStatement, requiredType, schemaContext, unqualifiedContext, } from './pikku-schema.types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
|
|
7
|
+
* runtime works without an audit table, and `KyselyAuditService.init()` applies
|
|
8
|
+
* this for the projects that wire the sink.
|
|
9
|
+
*/
|
|
10
|
+
export { auditSchema } from './audit.schema.js';
|
|
5
11
|
/**
|
|
6
12
|
* Every table the pikku runtime needs, in dependency order.
|
|
7
13
|
*
|
|
@@ -95,5 +101,12 @@ export declare const ensurePikkuSchema: (db: Kysely<any>, schema: PikkuSchema) =
|
|
|
95
101
|
* `types` comes from `resolveRequirements` against the database the migration
|
|
96
102
|
* is being written for. Omit it and foreign keys onto another source's columns
|
|
97
103
|
* compile as `text`, which is a shape, not a migration.
|
|
104
|
+
*
|
|
105
|
+
* `schema` qualifies every table the migration creates, for a project that
|
|
106
|
+
* keeps the runtime tables somewhere other than the default search path. It is
|
|
107
|
+
* applied here and not to `db` itself because the caller's connection is a
|
|
108
|
+
* throwaway the declaration was just *applied* to, in whatever schema that
|
|
109
|
+
* database actually has — qualifying that would create tables in a schema the
|
|
110
|
+
* scratch database has never heard of. Only the rendered SQL is bound.
|
|
98
111
|
*/
|
|
99
|
-
export declare const compilePikkuSchemas: (db: Kysely<any>, schemas?: PikkuSchema[], types?: RequiredTypes) => string;
|
|
112
|
+
export declare const compilePikkuSchemas: (db: Kysely<any>, schemas?: PikkuSchema[], types?: RequiredTypes, schemaName?: string) => string;
|
package/dist/src/schema/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CamelCasePlugin } from 'kysely';
|
|
2
|
+
import { schemaContext, } from './pikku-schema.types.js';
|
|
2
3
|
import { aiSchema } from './ai.schema.js';
|
|
3
4
|
import { channelSchema } from './channel.schema.js';
|
|
4
5
|
import { credentialSchema } from './credential.schema.js';
|
|
@@ -8,7 +9,13 @@ import { secretSchema } from './secret.schema.js';
|
|
|
8
9
|
import { sessionSchema } from './session.schema.js';
|
|
9
10
|
import { webhookSchema } from './webhook.schema.js';
|
|
10
11
|
import { workflowSchema } from './workflow.schema.js';
|
|
11
|
-
export { rawStatement, requiredType } from './pikku-schema.types.js';
|
|
12
|
+
export { rawStatement, requiredType, schemaContext, unqualifiedContext, } from './pikku-schema.types.js';
|
|
13
|
+
/**
|
|
14
|
+
* Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
|
|
15
|
+
* runtime works without an audit table, and `KyselyAuditService.init()` applies
|
|
16
|
+
* this for the projects that wire the sink.
|
|
17
|
+
*/
|
|
18
|
+
export { auditSchema } from './audit.schema.js';
|
|
12
19
|
/**
|
|
13
20
|
* Every table the pikku runtime needs, in dependency order.
|
|
14
21
|
*
|
|
@@ -37,6 +44,23 @@ export const pikkuSchemas = [
|
|
|
37
44
|
* silently creates quoted camelCase tables that nothing can read.
|
|
38
45
|
*/
|
|
39
46
|
const bind = (db) => db.withPlugin(new CamelCasePlugin());
|
|
47
|
+
/**
|
|
48
|
+
* The schema a connection is bound to, or `undefined` for one that is not.
|
|
49
|
+
*
|
|
50
|
+
* Read back out of compiled DDL rather than off the connection, because kysely
|
|
51
|
+
* keeps the `withSchema(...)` setting inside its executor and exposes no way to
|
|
52
|
+
* ask. Compiling a throwaway `create table` is the same trick `declaredTables`
|
|
53
|
+
* uses on the real statements, against the same regex — nothing is executed and
|
|
54
|
+
* the probe never leaves this function.
|
|
55
|
+
*
|
|
56
|
+
* Derived rather than passed in so that raw statements follow whatever
|
|
57
|
+
* connection they are handed. A caller that had to remember to say the schema
|
|
58
|
+
* twice would eventually say it once.
|
|
59
|
+
*/
|
|
60
|
+
const boundSchema = (db) => CREATE_TABLE.exec(db.schema.createTable('pikkuSchemaProbe').addColumn('id', 'text').compile()
|
|
61
|
+
.sql)?.[1];
|
|
62
|
+
/** The context for statements compiled against `db`, matching its binding. */
|
|
63
|
+
const contextFor = (db) => schemaContext(boundSchema(db));
|
|
40
64
|
/**
|
|
41
65
|
* Look up every prerequisite in `db`, reporting the gaps and the types found.
|
|
42
66
|
*
|
|
@@ -101,9 +125,10 @@ export const applyPikkuSchemas = async (db, schemas = pikkuSchemas) => {
|
|
|
101
125
|
}
|
|
102
126
|
await db.transaction().execute(async (trx) => {
|
|
103
127
|
const bound = bind(trx);
|
|
128
|
+
const ctx = contextFor(bound);
|
|
104
129
|
for (const schema of schemas) {
|
|
105
130
|
for (const statement of schema.statements) {
|
|
106
|
-
const query = statement(bound, types);
|
|
131
|
+
const query = statement(bound, types, ctx);
|
|
107
132
|
try {
|
|
108
133
|
await query.execute();
|
|
109
134
|
}
|
|
@@ -177,9 +202,10 @@ const CREATE_TABLE = /^\s*create table (?:if not exists\s+)?(?:"([^"]+)"\.)?"([^
|
|
|
177
202
|
*/
|
|
178
203
|
export const declaredTables = (schema, db) => {
|
|
179
204
|
const bound = bind(db);
|
|
205
|
+
const ctx = contextFor(bound);
|
|
180
206
|
const tables = [];
|
|
181
207
|
for (const statement of schema.statements) {
|
|
182
|
-
const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql);
|
|
208
|
+
const match = CREATE_TABLE.exec(statement(bound, {}, ctx).compile().sql);
|
|
183
209
|
if (match)
|
|
184
210
|
tables.push({ schema: match[1], name: match[2] });
|
|
185
211
|
}
|
|
@@ -265,10 +291,18 @@ export const ensurePikkuSchema = async (db, schema) => {
|
|
|
265
291
|
* `types` comes from `resolveRequirements` against the database the migration
|
|
266
292
|
* is being written for. Omit it and foreign keys onto another source's columns
|
|
267
293
|
* compile as `text`, which is a shape, not a migration.
|
|
294
|
+
*
|
|
295
|
+
* `schema` qualifies every table the migration creates, for a project that
|
|
296
|
+
* keeps the runtime tables somewhere other than the default search path. It is
|
|
297
|
+
* applied here and not to `db` itself because the caller's connection is a
|
|
298
|
+
* throwaway the declaration was just *applied* to, in whatever schema that
|
|
299
|
+
* database actually has — qualifying that would create tables in a schema the
|
|
300
|
+
* scratch database has never heard of. Only the rendered SQL is bound.
|
|
268
301
|
*/
|
|
269
|
-
export const compilePikkuSchemas = (db, schemas = pikkuSchemas, types = {}) => {
|
|
270
|
-
const bound = bind(db);
|
|
302
|
+
export const compilePikkuSchemas = (db, schemas = pikkuSchemas, types = {}, schemaName) => {
|
|
303
|
+
const bound = schemaName ? bind(db).withSchema(schemaName) : bind(db);
|
|
304
|
+
const ctx = contextFor(bound);
|
|
271
305
|
return schemas
|
|
272
|
-
.flatMap((schema) => schema.statements.map((statement) => `${statement(bound, types).compile().sql};`))
|
|
306
|
+
.flatMap((schema) => schema.statements.map((statement) => `${statement(bound, types, ctx).compile().sql};`))
|
|
273
307
|
.join('\n\n');
|
|
274
308
|
};
|