@pikku/kysely 0.13.7 → 0.13.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -0
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +3 -1
- package/dist/src/kysely-audit-service.d.ts +34 -2
- package/dist/src/kysely-audit-service.js +163 -3
- package/dist/src/kysely-secret-service.d.ts +4 -3
- package/dist/src/kysely-secret-service.js +6 -3
- package/dist/src/kysely-tables.d.ts +31 -0
- package/dist/src/kysely-virtual-user-run-store.d.ts +43 -0
- package/dist/src/kysely-virtual-user-run-store.js +131 -0
- package/dist/src/schema/audit.schema.d.ts +16 -0
- package/dist/src/schema/audit.schema.js +47 -0
- package/dist/src/schema/credential.schema.d.ts +3 -2
- package/dist/src/schema/credential.schema.js +4 -3
- package/dist/src/schema/index.d.ts +17 -4
- package/dist/src/schema/index.js +40 -6
- package/dist/src/schema/pikku-schema.types.d.ts +41 -5
- package/dist/src/schema/pikku-schema.types.js +27 -12
- package/dist/src/schema/virtual-user.schema.d.ts +19 -0
- package/dist/src/schema/virtual-user.schema.js +54 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/create-audited-kysely.test.ts +10 -10
- package/src/index.ts +7 -0
- package/src/kysely-audit-service.test.ts +360 -0
- package/src/kysely-audit-service.ts +181 -4
- package/src/kysely-secret-service.ts +11 -10
- package/src/kysely-services.test.ts +3 -3
- package/src/kysely-tables.test.ts +12 -1
- package/src/kysely-tables.ts +32 -0
- package/src/kysely-virtual-user-run-store.test.ts +212 -0
- package/src/kysely-virtual-user-run-store.ts +149 -0
- package/src/schema/audit.schema.ts +55 -0
- package/src/schema/credential.schema.ts +5 -3
- package/src/schema/index.ts +59 -10
- package/src/schema/pikku-schema.types.ts +65 -10
- package/src/schema/schema.test.ts +82 -4
- package/src/schema/virtual-user.schema.ts +65 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { parseJson } from './kysely-json.js';
|
|
3
|
+
import { ensurePikkuSchema } from './schema/index.js';
|
|
4
|
+
import { virtualUserSchema } from './schema/virtual-user.schema.js';
|
|
5
|
+
/**
|
|
6
|
+
* Records virtual-user runs in a `virtualUserRun` table.
|
|
7
|
+
*
|
|
8
|
+
* The store is the whole record of a run: `runVirtualUser` returns as soon as
|
|
9
|
+
* the row exists and the engine keeps going without it, so nothing else knows
|
|
10
|
+
* the run happened. That is why `fail()` exists as its own call — a run that
|
|
11
|
+
* crashed and a run that found nothing are different answers, and a row left at
|
|
12
|
+
* `running` is neither.
|
|
13
|
+
*
|
|
14
|
+
* Works with or without `SerializePlugin`, which not every project installs: the
|
|
15
|
+
* JSON columns are written as strings (which the plugin passes through) and read
|
|
16
|
+
* back through `parseJson` (which passes through what the plugin already
|
|
17
|
+
* parsed). Timestamps are written as ISO strings for the same reason — a bare
|
|
18
|
+
* SQLite driver cannot bind a `Date` at all.
|
|
19
|
+
*
|
|
20
|
+
* SECURITY: an `adversarial` run's findings are working exploits carrying live
|
|
21
|
+
* ids. This is a privileged store; the scaffolded reads are scope-gated, and
|
|
22
|
+
* exposing these rows more widely publishes your own exploits.
|
|
23
|
+
*/
|
|
24
|
+
export class KyselyVirtualUserRunStore {
|
|
25
|
+
db;
|
|
26
|
+
initialized = false;
|
|
27
|
+
constructor(db) {
|
|
28
|
+
this.db = db;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Creates the table on first use, like the audit sink — the runtime does not
|
|
32
|
+
* need it, so it arrives with the feature that fills it rather than in every
|
|
33
|
+
* database.
|
|
34
|
+
*/
|
|
35
|
+
async init() {
|
|
36
|
+
if (this.initialized)
|
|
37
|
+
return;
|
|
38
|
+
await ensurePikkuSchema(this.db, virtualUserSchema);
|
|
39
|
+
this.initialized = true;
|
|
40
|
+
}
|
|
41
|
+
async start(run) {
|
|
42
|
+
await this.init();
|
|
43
|
+
const runId = randomUUID();
|
|
44
|
+
await this.db
|
|
45
|
+
.insertInto('virtualUserRun')
|
|
46
|
+
.values({
|
|
47
|
+
runId,
|
|
48
|
+
persona: run.persona,
|
|
49
|
+
disposition: run.disposition,
|
|
50
|
+
// Text on every engine (see the schema), so the seed round-trips
|
|
51
|
+
// through drivers that hand a BIGINT back as a string.
|
|
52
|
+
seed: String(run.seed),
|
|
53
|
+
status: 'running',
|
|
54
|
+
goals: JSON.stringify(run.goals ?? []),
|
|
55
|
+
memory: JSON.stringify(run.memory ?? {}),
|
|
56
|
+
findings: '[]',
|
|
57
|
+
startedBy: run.startedBy ?? null,
|
|
58
|
+
})
|
|
59
|
+
.execute();
|
|
60
|
+
return runId;
|
|
61
|
+
}
|
|
62
|
+
async complete(runId, outcome) {
|
|
63
|
+
await this.init();
|
|
64
|
+
await this.db
|
|
65
|
+
.updateTable('virtualUserRun')
|
|
66
|
+
.set({
|
|
67
|
+
status: 'completed',
|
|
68
|
+
findings: JSON.stringify(outcome.findings),
|
|
69
|
+
tally: JSON.stringify(outcome.tally),
|
|
70
|
+
// Overwritten rather than merged: the engine's memory already carries
|
|
71
|
+
// what it was given, plus what it learned on the way.
|
|
72
|
+
memory: JSON.stringify(outcome.memory),
|
|
73
|
+
stoppedBy: outcome.stoppedBy,
|
|
74
|
+
finishedAt: new Date().toISOString(),
|
|
75
|
+
})
|
|
76
|
+
.where('runId', '=', runId)
|
|
77
|
+
.execute();
|
|
78
|
+
}
|
|
79
|
+
async fail(runId, error) {
|
|
80
|
+
await this.init();
|
|
81
|
+
await this.db
|
|
82
|
+
.updateTable('virtualUserRun')
|
|
83
|
+
.set({
|
|
84
|
+
status: 'failed',
|
|
85
|
+
error,
|
|
86
|
+
finishedAt: new Date().toISOString(),
|
|
87
|
+
})
|
|
88
|
+
.where('runId', '=', runId)
|
|
89
|
+
.execute();
|
|
90
|
+
}
|
|
91
|
+
async get(runId) {
|
|
92
|
+
await this.init();
|
|
93
|
+
const row = await this.db
|
|
94
|
+
.selectFrom('virtualUserRun')
|
|
95
|
+
.selectAll()
|
|
96
|
+
.where('runId', '=', runId)
|
|
97
|
+
.executeTakeFirst();
|
|
98
|
+
return row ? this.toRecord(row) : null;
|
|
99
|
+
}
|
|
100
|
+
async list(options) {
|
|
101
|
+
await this.init();
|
|
102
|
+
let query = this.db.selectFrom('virtualUserRun').selectAll();
|
|
103
|
+
if (options?.persona) {
|
|
104
|
+
query = query.where('persona', '=', options.persona);
|
|
105
|
+
}
|
|
106
|
+
const rows = await query
|
|
107
|
+
.orderBy('createdAt', 'desc')
|
|
108
|
+
.limit(options?.limit ?? 50)
|
|
109
|
+
.offset(options?.offset ?? 0)
|
|
110
|
+
.execute();
|
|
111
|
+
return rows.map((row) => this.toRecord(row));
|
|
112
|
+
}
|
|
113
|
+
toRecord(row) {
|
|
114
|
+
return {
|
|
115
|
+
runId: row.runId,
|
|
116
|
+
persona: row.persona,
|
|
117
|
+
disposition: row.disposition,
|
|
118
|
+
seed: Number(row.seed),
|
|
119
|
+
status: row.status,
|
|
120
|
+
goals: parseJson(row.goals) ?? [],
|
|
121
|
+
memory: parseJson(row.memory) ?? {},
|
|
122
|
+
findings: parseJson(row.findings) ?? [],
|
|
123
|
+
tally: row.tally ? (parseJson(row.tally) ?? null) : null,
|
|
124
|
+
stoppedBy: row.stoppedBy,
|
|
125
|
+
error: row.error,
|
|
126
|
+
startedBy: row.startedBy,
|
|
127
|
+
createdAt: new Date(row.createdAt),
|
|
128
|
+
finishedAt: row.finishedAt ? new Date(row.finishedAt) : null,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { PikkuSchema } from './pikku-schema.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
4
|
+
*
|
|
5
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
6
|
+
* a project that has wired a durable audit sink does, and creating it for
|
|
7
|
+
* everyone would put an empty table in every database. `KyselyAuditService.init()`
|
|
8
|
+
* applies it, so it arrives exactly when the sink that fills it does.
|
|
9
|
+
*
|
|
10
|
+
* Every column is text on every engine, matching the platform audit-queue
|
|
11
|
+
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
12
|
+
* the same reader can read. `occurredAt` is an ISO 8601 string rather than a
|
|
13
|
+
* timestamp for that reason: string ordering is chronological ordering, and the
|
|
14
|
+
* queue consumer has no shared type to agree on.
|
|
15
|
+
*/
|
|
16
|
+
export declare const auditSchema: PikkuSchema;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
5
|
+
* a project that has wired a durable audit sink does, and creating it for
|
|
6
|
+
* everyone would put an empty table in every database. `KyselyAuditService.init()`
|
|
7
|
+
* applies it, so it arrives exactly when the sink that fills it does.
|
|
8
|
+
*
|
|
9
|
+
* Every column is text on every engine, matching the platform audit-queue
|
|
10
|
+
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
11
|
+
* the same reader can read. `occurredAt` is an ISO 8601 string rather than a
|
|
12
|
+
* timestamp for that reason: string ordering is chronological ordering, and the
|
|
13
|
+
* queue consumer has no shared type to agree on.
|
|
14
|
+
*/
|
|
15
|
+
export const auditSchema = {
|
|
16
|
+
name: 'audit',
|
|
17
|
+
statements: [
|
|
18
|
+
(db) => db.schema
|
|
19
|
+
.createTable('audit')
|
|
20
|
+
.addColumn('auditId', 'text', (col) => col.primaryKey())
|
|
21
|
+
.addColumn('occurredAt', 'text', (col) => col.notNull())
|
|
22
|
+
.addColumn('type', 'text', (col) => col.notNull())
|
|
23
|
+
.addColumn('source', 'text', (col) => col.defaultTo('auto').notNull())
|
|
24
|
+
.addColumn('outcome', 'text')
|
|
25
|
+
.addColumn('functionId', 'text')
|
|
26
|
+
.addColumn('wireType', 'text')
|
|
27
|
+
.addColumn('traceId', 'text')
|
|
28
|
+
.addColumn('transactionId', 'text')
|
|
29
|
+
.addColumn('queryId', 'text')
|
|
30
|
+
.addColumn('userId', 'text')
|
|
31
|
+
.addColumn('orgId', 'text')
|
|
32
|
+
.addColumn('pikkuUserId', 'text')
|
|
33
|
+
.addColumn('tables', 'text')
|
|
34
|
+
.addColumn('changedCols', 'text')
|
|
35
|
+
.addColumn('event', 'text')
|
|
36
|
+
.addColumn('old', 'text')
|
|
37
|
+
.addColumn('data', 'text'),
|
|
38
|
+
// The trail is only ever read newest-first, and the two filters the console
|
|
39
|
+
// offers are user and type.
|
|
40
|
+
(db) => db.schema
|
|
41
|
+
.createIndex('idx_audit_occurred_at')
|
|
42
|
+
.on('audit')
|
|
43
|
+
.column('occurredAt'),
|
|
44
|
+
(db) => db.schema.createIndex('idx_audit_user_id').on('audit').column('userId'),
|
|
45
|
+
(db) => db.schema.createIndex('idx_audit_type').on('audit').column('type'),
|
|
46
|
+
],
|
|
47
|
+
};
|
|
@@ -5,7 +5,8 @@ import { type PikkuSchema } from './pikku-schema.types.js';
|
|
|
5
5
|
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
6
6
|
* rows with a null user must still collide on name — which the schema builder
|
|
7
7
|
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
8
|
-
* which is why the table
|
|
9
|
-
* path.
|
|
8
|
+
* which is why the table goes through `ctx.table` rather than being left to the
|
|
9
|
+
* search path. The index's own name stays unqualified: an index belongs to the
|
|
10
|
+
* schema of the table it is on, and postgres rejects one that says otherwise.
|
|
10
11
|
*/
|
|
11
12
|
export declare const credentialSchema: PikkuSchema;
|
|
@@ -6,8 +6,9 @@ import { rawStatement } from './pikku-schema.types.js';
|
|
|
6
6
|
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
7
7
|
* rows with a null user must still collide on name — which the schema builder
|
|
8
8
|
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
9
|
-
* which is why the table
|
|
10
|
-
* path.
|
|
9
|
+
* which is why the table goes through `ctx.table` rather than being left to the
|
|
10
|
+
* search path. The index's own name stays unqualified: an index belongs to the
|
|
11
|
+
* schema of the table it is on, and postgres rejects one that says otherwise.
|
|
11
12
|
*/
|
|
12
13
|
export const credentialSchema = {
|
|
13
14
|
name: 'credential',
|
|
@@ -21,7 +22,7 @@ export const credentialSchema = {
|
|
|
21
22
|
.addColumn('keyVersion', 'integer', (col) => col.notNull())
|
|
22
23
|
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
23
24
|
.addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
|
|
24
|
-
rawStatement(sql `CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`),
|
|
25
|
+
rawStatement(({ table }) => sql `CREATE UNIQUE INDEX credentials_name_user_id_unique ON ${table('credentials')} (name, COALESCE(user_id, ''))`),
|
|
25
26
|
(db) => db.schema
|
|
26
27
|
.createTable('credentialKekSalts')
|
|
27
28
|
.addColumn('keyVersion', 'integer', (col) => col.primaryKey())
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { type Kysely } from 'kysely';
|
|
2
|
-
import type
|
|
3
|
-
export type { PikkuSchema, RequiredTypes, SchemaRequirement, SchemaStatement, SchemaStatementFactory, } from './pikku-schema.types.js';
|
|
4
|
-
export { rawStatement, requiredType } from './pikku-schema.types.js';
|
|
2
|
+
import { type PikkuSchema, type RequiredTypes, type SchemaRequirement } from './pikku-schema.types.js';
|
|
3
|
+
export type { PikkuSchema, RequiredTypes, SchemaContext, SchemaRequirement, SchemaStatement, SchemaStatementFactory, } from './pikku-schema.types.js';
|
|
4
|
+
export { rawStatement, requiredType, schemaContext, unqualifiedContext, } from './pikku-schema.types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
|
|
7
|
+
* runtime works without an audit table, and `KyselyAuditService.init()` applies
|
|
8
|
+
* this for the projects that wire the sink.
|
|
9
|
+
*/
|
|
10
|
+
export { auditSchema } from './audit.schema.js';
|
|
5
11
|
/**
|
|
6
12
|
* Every table the pikku runtime needs, in dependency order.
|
|
7
13
|
*
|
|
@@ -95,5 +101,12 @@ export declare const ensurePikkuSchema: (db: Kysely<any>, schema: PikkuSchema) =
|
|
|
95
101
|
* `types` comes from `resolveRequirements` against the database the migration
|
|
96
102
|
* is being written for. Omit it and foreign keys onto another source's columns
|
|
97
103
|
* compile as `text`, which is a shape, not a migration.
|
|
104
|
+
*
|
|
105
|
+
* `schema` qualifies every table the migration creates, for a project that
|
|
106
|
+
* keeps the runtime tables somewhere other than the default search path. It is
|
|
107
|
+
* applied here and not to `db` itself because the caller's connection is a
|
|
108
|
+
* throwaway the declaration was just *applied* to, in whatever schema that
|
|
109
|
+
* database actually has — qualifying that would create tables in a schema the
|
|
110
|
+
* scratch database has never heard of. Only the rendered SQL is bound.
|
|
98
111
|
*/
|
|
99
|
-
export declare const compilePikkuSchemas: (db: Kysely<any>, schemas?: PikkuSchema[], types?: RequiredTypes) => string;
|
|
112
|
+
export declare const compilePikkuSchemas: (db: Kysely<any>, schemas?: PikkuSchema[], types?: RequiredTypes, schemaName?: string) => string;
|
package/dist/src/schema/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CamelCasePlugin } from 'kysely';
|
|
2
|
+
import { schemaContext, } from './pikku-schema.types.js';
|
|
2
3
|
import { aiSchema } from './ai.schema.js';
|
|
3
4
|
import { channelSchema } from './channel.schema.js';
|
|
4
5
|
import { credentialSchema } from './credential.schema.js';
|
|
@@ -8,7 +9,13 @@ import { secretSchema } from './secret.schema.js';
|
|
|
8
9
|
import { sessionSchema } from './session.schema.js';
|
|
9
10
|
import { webhookSchema } from './webhook.schema.js';
|
|
10
11
|
import { workflowSchema } from './workflow.schema.js';
|
|
11
|
-
export { rawStatement, requiredType } from './pikku-schema.types.js';
|
|
12
|
+
export { rawStatement, requiredType, schemaContext, unqualifiedContext, } from './pikku-schema.types.js';
|
|
13
|
+
/**
|
|
14
|
+
* Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
|
|
15
|
+
* runtime works without an audit table, and `KyselyAuditService.init()` applies
|
|
16
|
+
* this for the projects that wire the sink.
|
|
17
|
+
*/
|
|
18
|
+
export { auditSchema } from './audit.schema.js';
|
|
12
19
|
/**
|
|
13
20
|
* Every table the pikku runtime needs, in dependency order.
|
|
14
21
|
*
|
|
@@ -37,6 +44,23 @@ export const pikkuSchemas = [
|
|
|
37
44
|
* silently creates quoted camelCase tables that nothing can read.
|
|
38
45
|
*/
|
|
39
46
|
const bind = (db) => db.withPlugin(new CamelCasePlugin());
|
|
47
|
+
/**
|
|
48
|
+
* The schema a connection is bound to, or `undefined` for one that is not.
|
|
49
|
+
*
|
|
50
|
+
* Read back out of compiled DDL rather than off the connection, because kysely
|
|
51
|
+
* keeps the `withSchema(...)` setting inside its executor and exposes no way to
|
|
52
|
+
* ask. Compiling a throwaway `create table` is the same trick `declaredTables`
|
|
53
|
+
* uses on the real statements, against the same regex — nothing is executed and
|
|
54
|
+
* the probe never leaves this function.
|
|
55
|
+
*
|
|
56
|
+
* Derived rather than passed in so that raw statements follow whatever
|
|
57
|
+
* connection they are handed. A caller that had to remember to say the schema
|
|
58
|
+
* twice would eventually say it once.
|
|
59
|
+
*/
|
|
60
|
+
const boundSchema = (db) => CREATE_TABLE.exec(db.schema.createTable('pikkuSchemaProbe').addColumn('id', 'text').compile()
|
|
61
|
+
.sql)?.[1];
|
|
62
|
+
/** The context for statements compiled against `db`, matching its binding. */
|
|
63
|
+
const contextFor = (db) => schemaContext(boundSchema(db));
|
|
40
64
|
/**
|
|
41
65
|
* Look up every prerequisite in `db`, reporting the gaps and the types found.
|
|
42
66
|
*
|
|
@@ -101,9 +125,10 @@ export const applyPikkuSchemas = async (db, schemas = pikkuSchemas) => {
|
|
|
101
125
|
}
|
|
102
126
|
await db.transaction().execute(async (trx) => {
|
|
103
127
|
const bound = bind(trx);
|
|
128
|
+
const ctx = contextFor(bound);
|
|
104
129
|
for (const schema of schemas) {
|
|
105
130
|
for (const statement of schema.statements) {
|
|
106
|
-
const query = statement(bound, types);
|
|
131
|
+
const query = statement(bound, types, ctx);
|
|
107
132
|
try {
|
|
108
133
|
await query.execute();
|
|
109
134
|
}
|
|
@@ -177,9 +202,10 @@ const CREATE_TABLE = /^\s*create table (?:if not exists\s+)?(?:"([^"]+)"\.)?"([^
|
|
|
177
202
|
*/
|
|
178
203
|
export const declaredTables = (schema, db) => {
|
|
179
204
|
const bound = bind(db);
|
|
205
|
+
const ctx = contextFor(bound);
|
|
180
206
|
const tables = [];
|
|
181
207
|
for (const statement of schema.statements) {
|
|
182
|
-
const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql);
|
|
208
|
+
const match = CREATE_TABLE.exec(statement(bound, {}, ctx).compile().sql);
|
|
183
209
|
if (match)
|
|
184
210
|
tables.push({ schema: match[1], name: match[2] });
|
|
185
211
|
}
|
|
@@ -265,10 +291,18 @@ export const ensurePikkuSchema = async (db, schema) => {
|
|
|
265
291
|
* `types` comes from `resolveRequirements` against the database the migration
|
|
266
292
|
* is being written for. Omit it and foreign keys onto another source's columns
|
|
267
293
|
* compile as `text`, which is a shape, not a migration.
|
|
294
|
+
*
|
|
295
|
+
* `schema` qualifies every table the migration creates, for a project that
|
|
296
|
+
* keeps the runtime tables somewhere other than the default search path. It is
|
|
297
|
+
* applied here and not to `db` itself because the caller's connection is a
|
|
298
|
+
* throwaway the declaration was just *applied* to, in whatever schema that
|
|
299
|
+
* database actually has — qualifying that would create tables in a schema the
|
|
300
|
+
* scratch database has never heard of. Only the rendered SQL is bound.
|
|
268
301
|
*/
|
|
269
|
-
export const compilePikkuSchemas = (db, schemas = pikkuSchemas, types = {}) => {
|
|
270
|
-
const bound = bind(db);
|
|
302
|
+
export const compilePikkuSchemas = (db, schemas = pikkuSchemas, types = {}, schemaName) => {
|
|
303
|
+
const bound = schemaName ? bind(db).withSchema(schemaName) : bind(db);
|
|
304
|
+
const ctx = contextFor(bound);
|
|
271
305
|
return schemas
|
|
272
|
-
.flatMap((schema) => schema.statements.map((statement) => `${statement(bound, types).compile().sql};`))
|
|
306
|
+
.flatMap((schema) => schema.statements.map((statement) => `${statement(bound, types, ctx).compile().sql};`))
|
|
273
307
|
.join('\n\n');
|
|
274
308
|
};
|
|
@@ -11,7 +11,40 @@ export interface SchemaStatement {
|
|
|
11
11
|
execute(): Promise<unknown>;
|
|
12
12
|
compile(): CompiledQuery;
|
|
13
13
|
}
|
|
14
|
-
export type SchemaStatementFactory = (db: Kysely<any>, types: RequiredTypes) => SchemaStatement;
|
|
14
|
+
export type SchemaStatementFactory = (db: Kysely<any>, types: RequiredTypes, ctx: SchemaContext) => SchemaStatement;
|
|
15
|
+
/**
|
|
16
|
+
* What a statement needs to know about the schema its DDL lands in.
|
|
17
|
+
*
|
|
18
|
+
* Only raw SQL needs this. The schema builder learns the schema from the
|
|
19
|
+
* connection — `withSchema('app')` qualifies every table it names, foreign key
|
|
20
|
+
* targets included — but a raw fragment is passed through verbatim, so a
|
|
21
|
+
* statement that spells its own SQL has to spell the qualifier too.
|
|
22
|
+
*/
|
|
23
|
+
export interface SchemaContext {
|
|
24
|
+
/**
|
|
25
|
+
* A table as the surrounding DDL addresses it: `"app"."credentials"` on a
|
|
26
|
+
* schema-bound connection, `"credentials"` otherwise.
|
|
27
|
+
*
|
|
28
|
+
* Takes the physical name, not the camelCase one the schema builder accepts.
|
|
29
|
+
* Raw SQL does not go through `CamelCasePlugin`, so it is already written in
|
|
30
|
+
* physical names throughout and this is no exception.
|
|
31
|
+
*/
|
|
32
|
+
table: (name: string) => RawBuilder<unknown>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The `SchemaContext` for DDL that names no schema.
|
|
36
|
+
*
|
|
37
|
+
* The default everywhere a schema has not been asked for, which is every engine
|
|
38
|
+
* but postgres and most postgres projects too.
|
|
39
|
+
*/
|
|
40
|
+
export declare const unqualifiedContext: SchemaContext;
|
|
41
|
+
/**
|
|
42
|
+
* A `SchemaContext` qualifying every table into `schema`.
|
|
43
|
+
*
|
|
44
|
+
* Identifiers are quoted rather than interpolated bare so a schema named after
|
|
45
|
+
* a reserved word still compiles.
|
|
46
|
+
*/
|
|
47
|
+
export declare const schemaContext: (schema: string | undefined) => SchemaContext;
|
|
15
48
|
/**
|
|
16
49
|
* The physical types of the columns named in `requires`, keyed `table.column`.
|
|
17
50
|
*
|
|
@@ -68,8 +101,11 @@ export interface SchemaRequirement {
|
|
|
68
101
|
* Wrap a raw SQL fragment as a statement.
|
|
69
102
|
*
|
|
70
103
|
* For the DDL kysely's schema builder cannot express — a unique index over an
|
|
71
|
-
* expression, say.
|
|
72
|
-
*
|
|
73
|
-
*
|
|
104
|
+
* expression, say. Raw SQL is *not* rewritten by `withSchema`, so a fragment
|
|
105
|
+
* naming a table unqualified resolves against the connection's search path
|
|
106
|
+
* rather than the schema the rest of the declaration lands in. Take the
|
|
107
|
+
* function form and name tables through `ctx.table` to stay with them; the
|
|
108
|
+
* bare-fragment form is for SQL that names no table at all.
|
|
74
109
|
*/
|
|
75
|
-
export declare
|
|
110
|
+
export declare function rawStatement(fragment: RawBuilder<unknown>): SchemaStatementFactory;
|
|
111
|
+
export declare function rawStatement(build: (ctx: SchemaContext) => RawBuilder<unknown>): SchemaStatementFactory;
|
|
@@ -1,4 +1,22 @@
|
|
|
1
1
|
import { sql } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* The `SchemaContext` for DDL that names no schema.
|
|
4
|
+
*
|
|
5
|
+
* The default everywhere a schema has not been asked for, which is every engine
|
|
6
|
+
* but postgres and most postgres projects too.
|
|
7
|
+
*/
|
|
8
|
+
export const unqualifiedContext = {
|
|
9
|
+
table: (name) => sql.raw(`"${name}"`),
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* A `SchemaContext` qualifying every table into `schema`.
|
|
13
|
+
*
|
|
14
|
+
* Identifiers are quoted rather than interpolated bare so a schema named after
|
|
15
|
+
* a reserved word still compiles.
|
|
16
|
+
*/
|
|
17
|
+
export const schemaContext = (schema) => schema
|
|
18
|
+
? { table: (name) => sql.raw(`"${schema}"."${name}"`) }
|
|
19
|
+
: unqualifiedContext;
|
|
2
20
|
/**
|
|
3
21
|
* The declared type for a foreign key onto a column another source owns.
|
|
4
22
|
*
|
|
@@ -7,15 +25,12 @@ import { sql } from 'kysely';
|
|
|
7
25
|
* and only the shape of the statement is being inspected.
|
|
8
26
|
*/
|
|
9
27
|
export const requiredType = (types, table, column) => sql.raw(types[`${table}.${column}`] ?? 'text');
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
execute: () => fragment.execute(db),
|
|
20
|
-
compile: () => fragment.compile(db),
|
|
21
|
-
});
|
|
28
|
+
export function rawStatement(source) {
|
|
29
|
+
return (db, _types, ctx = unqualifiedContext) => {
|
|
30
|
+
const fragment = typeof source === 'function' ? source(ctx) : source;
|
|
31
|
+
return {
|
|
32
|
+
execute: () => fragment.execute(db),
|
|
33
|
+
compile: () => fragment.compile(db),
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { PikkuSchema } from './pikku-schema.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
|
|
4
|
+
*
|
|
5
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
6
|
+
* a project that runs virtual users does, and creating it for everyone would
|
|
7
|
+
* put an empty table in every database. `KyselyVirtualUserRunStore.init()`
|
|
8
|
+
* applies it, so it arrives exactly when the thing that fills it does.
|
|
9
|
+
*
|
|
10
|
+
* `persona` is text rather than a foreign key: personas are declared in code
|
|
11
|
+
* with `definePersonas()`, not rows, and a run has to outlive the deletion of
|
|
12
|
+
* the persona it ran as.
|
|
13
|
+
*
|
|
14
|
+
* The JSON columns are text on every engine, like the audit table's, because
|
|
15
|
+
* the store serialises them itself — a project on SQLite and one on Postgres
|
|
16
|
+
* then hold the same bytes, and comparing this week's findings against last
|
|
17
|
+
* week's does not depend on which database they were found in.
|
|
18
|
+
*/
|
|
19
|
+
export declare const virtualUserSchema: PikkuSchema;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { sql } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
|
|
4
|
+
*
|
|
5
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
6
|
+
* a project that runs virtual users does, and creating it for everyone would
|
|
7
|
+
* put an empty table in every database. `KyselyVirtualUserRunStore.init()`
|
|
8
|
+
* applies it, so it arrives exactly when the thing that fills it does.
|
|
9
|
+
*
|
|
10
|
+
* `persona` is text rather than a foreign key: personas are declared in code
|
|
11
|
+
* with `definePersonas()`, not rows, and a run has to outlive the deletion of
|
|
12
|
+
* the persona it ran as.
|
|
13
|
+
*
|
|
14
|
+
* The JSON columns are text on every engine, like the audit table's, because
|
|
15
|
+
* the store serialises them itself — a project on SQLite and one on Postgres
|
|
16
|
+
* then hold the same bytes, and comparing this week's findings against last
|
|
17
|
+
* week's does not depend on which database they were found in.
|
|
18
|
+
*/
|
|
19
|
+
export const virtualUserSchema = {
|
|
20
|
+
name: 'virtualUser',
|
|
21
|
+
statements: [
|
|
22
|
+
(db) => db.schema
|
|
23
|
+
.createTable('virtualUserRun')
|
|
24
|
+
.addColumn('runId', 'varchar(36)', (col) => col.primaryKey())
|
|
25
|
+
.addColumn('persona', 'varchar(255)', (col) => col.notNull())
|
|
26
|
+
.addColumn('disposition', 'varchar(50)', (col) => col.notNull())
|
|
27
|
+
.addColumn('seed', 'bigint', (col) => col.notNull())
|
|
28
|
+
// running | completed | failed. Not derived from `finishedAt` being
|
|
29
|
+
// null: a crashed run has no finish time either, and "crashed" and
|
|
30
|
+
// "still going" are not the same answer.
|
|
31
|
+
.addColumn('status', 'varchar(50)', (col) => col.defaultTo('running').notNull())
|
|
32
|
+
.addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
|
|
33
|
+
.addColumn('memory', 'text', (col) => col.defaultTo('{}').notNull())
|
|
34
|
+
.addColumn('findings', 'text', (col) => col.defaultTo('[]').notNull())
|
|
35
|
+
// Counts the engine kept: steps, calls, mutations, tokens, elapsed.
|
|
36
|
+
.addColumn('tally', 'text')
|
|
37
|
+
.addColumn('stoppedBy', 'varchar(255)')
|
|
38
|
+
// Why the run itself failed, as opposed to what it found.
|
|
39
|
+
.addColumn('error', 'text')
|
|
40
|
+
.addColumn('startedBy', 'varchar(255)')
|
|
41
|
+
.addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
|
|
42
|
+
.addColumn('finishedAt', 'timestamp'),
|
|
43
|
+
// The two reads: the run list newest-first, and one persona's history when
|
|
44
|
+
// a finding needs comparing against what that persona found before.
|
|
45
|
+
(db) => db.schema
|
|
46
|
+
.createIndex('idx_virtual_user_run_created')
|
|
47
|
+
.on('virtualUserRun')
|
|
48
|
+
.column('createdAt'),
|
|
49
|
+
(db) => db.schema
|
|
50
|
+
.createIndex('idx_virtual_user_run_persona')
|
|
51
|
+
.on('virtualUserRun')
|
|
52
|
+
.columns(['persona', 'createdAt']),
|
|
53
|
+
],
|
|
54
|
+
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-ai-agent-run-service.ts","../src/kysely-ai-run-state-service.ts","../src/kysely-ai-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/ai.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}
|
|
1
|
+
{"root":["../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-ai-agent-run-service.ts","../src/kysely-ai-run-state-service.ts","../src/kysely-ai-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-virtual-user-run-store.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/ai.schema.ts","../src/schema/audit.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/virtual-user.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/kysely",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.10",
|
|
4
4
|
"description": "Kysely-backed Pikku services — workflow state, secrets, sessions, scopes and audit",
|
|
5
5
|
"author": "yasser.fadl@gmail.com",
|
|
6
6
|
"license": "MIT",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"prepublishOnly": "yarn build"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@pikku/core": "^0.12.
|
|
24
|
+
"@pikku/core": "^0.12.77"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"kysely": "^0.29.0"
|
|
@@ -33,8 +33,8 @@ interface AuditEventTable {
|
|
|
33
33
|
traceId: string | null
|
|
34
34
|
transactionId: string | null
|
|
35
35
|
queryId: string | null
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
userId: string | null
|
|
37
|
+
orgId: string | null
|
|
38
38
|
actorPikkuUserId: string | null
|
|
39
39
|
metadataJson: string | null
|
|
40
40
|
}
|
|
@@ -76,8 +76,8 @@ async function createAuditEventsTable(
|
|
|
76
76
|
.addColumn('traceId', 'text')
|
|
77
77
|
.addColumn('transactionId', 'text')
|
|
78
78
|
.addColumn('queryId', 'text')
|
|
79
|
-
.addColumn('
|
|
80
|
-
.addColumn('
|
|
79
|
+
.addColumn('userId', 'text')
|
|
80
|
+
.addColumn('orgId', 'text')
|
|
81
81
|
.addColumn('actorPikkuUserId', 'text')
|
|
82
82
|
.addColumn('metadataJson', 'text')
|
|
83
83
|
.execute()
|
|
@@ -113,9 +113,9 @@ class ImmediateAuditService implements AuditService {
|
|
|
113
113
|
traceId: event.traceId ?? null,
|
|
114
114
|
transactionId: event.transactionId ?? null,
|
|
115
115
|
queryId: event.queryId ?? null,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
actorPikkuUserId: event.
|
|
116
|
+
userId: event.userIdentity?.userId ?? null,
|
|
117
|
+
orgId: event.userIdentity?.orgId ?? null,
|
|
118
|
+
actorPikkuUserId: event.userIdentity?.pikkuUserId ?? null,
|
|
119
119
|
metadataJson: event.metadata ? JSON.stringify(event.metadata) : null,
|
|
120
120
|
})
|
|
121
121
|
.execute()
|
|
@@ -207,7 +207,7 @@ describe('createAuditedKysely', () => {
|
|
|
207
207
|
changes: { title: 'write tests', done: 0 },
|
|
208
208
|
rowCount: 1,
|
|
209
209
|
})
|
|
210
|
-
assert.deepEqual(events[0]?.
|
|
210
|
+
assert.deepEqual(events[0]?.userIdentity, {
|
|
211
211
|
userId: 'user-1',
|
|
212
212
|
orgId: 'org-1',
|
|
213
213
|
pikkuUserId: 'user-1',
|
|
@@ -257,7 +257,7 @@ describe('createAuditedKysely', () => {
|
|
|
257
257
|
title: 'after',
|
|
258
258
|
done: 1,
|
|
259
259
|
})
|
|
260
|
-
assert.deepEqual(events[0]?.
|
|
260
|
+
assert.deepEqual(events[0]?.userIdentity, {
|
|
261
261
|
userId: 'user-2',
|
|
262
262
|
orgId: undefined,
|
|
263
263
|
pikkuUserId: 'user-2',
|
|
@@ -455,7 +455,7 @@ describe('createAuditedKysely', () => {
|
|
|
455
455
|
['todos:best-effort', 'todos:best-effort']
|
|
456
456
|
)
|
|
457
457
|
assert.deepEqual(
|
|
458
|
-
persisted.map((event) => event.
|
|
458
|
+
persisted.map((event) => event.userId),
|
|
459
459
|
['user-best-effort', 'user-best-effort']
|
|
460
460
|
)
|
|
461
461
|
assert.equal(persisted[0]?.queryId, 'q-1')
|
package/src/index.ts
CHANGED
|
@@ -19,6 +19,7 @@ export {
|
|
|
19
19
|
type CreateAuditedKyselyOptions,
|
|
20
20
|
} from './create-audited-kysely.js'
|
|
21
21
|
export { KyselyAuditService } from './kysely-audit-service.js'
|
|
22
|
+
export { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
|
|
22
23
|
|
|
23
24
|
export {
|
|
24
25
|
SerializePlugin,
|
|
@@ -34,7 +35,9 @@ export {
|
|
|
34
35
|
type SerializePluginOptions,
|
|
35
36
|
} from './serialize-plugin.js'
|
|
36
37
|
|
|
38
|
+
export { virtualUserSchema } from './schema/virtual-user.schema.js'
|
|
37
39
|
export {
|
|
40
|
+
auditSchema,
|
|
38
41
|
pikkuSchemas,
|
|
39
42
|
applyPikkuSchemas,
|
|
40
43
|
compilePikkuSchemas,
|
|
@@ -55,3 +58,7 @@ export {
|
|
|
55
58
|
export type { KyselyPikkuDB } from './kysely-tables.js'
|
|
56
59
|
export type { WorkflowRunService } from '@pikku/core/workflow'
|
|
57
60
|
export type { AgentRunService, AgentRunRow } from '@pikku/core/ai-agent'
|
|
61
|
+
export type {
|
|
62
|
+
VirtualUserRunStore,
|
|
63
|
+
VirtualUserRunRecord,
|
|
64
|
+
} from '@pikku/core/virtual-user'
|