@pikku/kysely 0.13.6 → 0.13.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +505 -0
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +3 -1
- package/dist/src/kysely-audit-service.d.ts +34 -2
- package/dist/src/kysely-audit-service.js +163 -3
- package/dist/src/kysely-credential-service.d.ts +3 -0
- package/dist/src/kysely-credential-service.js +50 -11
- package/dist/src/kysely-scope-service.d.ts +38 -0
- package/dist/src/kysely-scope-service.js +125 -2
- package/dist/src/kysely-secret-service.d.ts +7 -3
- package/dist/src/kysely-secret-service.js +55 -13
- package/dist/src/kysely-tables.d.ts +49 -2
- package/dist/src/kysely-virtual-user-run-store.d.ts +43 -0
- package/dist/src/kysely-virtual-user-run-store.js +131 -0
- package/dist/src/kysely-workflow-service.d.ts +0 -1
- package/dist/src/kysely-workflow-service.js +0 -12
- package/dist/src/schema/audit.schema.d.ts +16 -0
- package/dist/src/schema/audit.schema.js +47 -0
- package/dist/src/schema/credential.schema.d.ts +3 -2
- package/dist/src/schema/credential.schema.js +9 -3
- package/dist/src/schema/index.d.ts +17 -4
- package/dist/src/schema/index.js +40 -6
- package/dist/src/schema/pikku-schema.types.d.ts +41 -5
- package/dist/src/schema/pikku-schema.types.js +27 -12
- package/dist/src/schema/scope.schema.js +12 -0
- package/dist/src/schema/secret.schema.js +5 -0
- package/dist/src/schema/virtual-user.schema.d.ts +19 -0
- package/dist/src/schema/virtual-user.schema.js +54 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/create-audited-kysely.test.ts +10 -10
- package/src/index.ts +7 -0
- package/src/kysely-audit-service.test.ts +360 -0
- package/src/kysely-audit-service.ts +181 -4
- package/src/kysely-credential-service.ts +57 -12
- package/src/kysely-scope-service.ts +151 -2
- package/src/kysely-secret-service.ts +70 -21
- package/src/kysely-services.test.ts +123 -1
- package/src/kysely-tables.test.ts +12 -1
- package/src/kysely-tables.ts +53 -2
- package/src/kysely-virtual-user-run-store.test.ts +212 -0
- package/src/kysely-virtual-user-run-store.ts +149 -0
- package/src/kysely-workflow-service.ts +0 -17
- package/src/schema/audit.schema.ts +55 -0
- package/src/schema/credential.schema.ts +14 -3
- package/src/schema/index.ts +59 -10
- package/src/schema/pikku-schema.types.ts +65 -10
- package/src/schema/schema.test.ts +82 -4
- package/src/schema/scope.schema.ts +14 -0
- package/src/schema/secret.schema.ts +9 -0
- package/src/schema/virtual-user.schema.ts +65 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { describe, test, beforeEach } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import Database from 'better-sqlite3'
|
|
4
|
+
import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
|
|
5
|
+
|
|
6
|
+
import { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
|
|
7
|
+
import { SerializePlugin } from './serialize-plugin.js'
|
|
8
|
+
import type { KyselyPikkuDB } from './kysely-tables.js'
|
|
9
|
+
|
|
10
|
+
// SerializePlugin is here because most projects in this package run with it, and
|
|
11
|
+
// it is the harder case: it deserialises JSON columns the store has already
|
|
12
|
+
// stringified. The store has to survive that, and its absence.
|
|
13
|
+
const newDb = () =>
|
|
14
|
+
new Kysely<KyselyPikkuDB>({
|
|
15
|
+
dialect: new SqliteDialect({ database: new Database(':memory:') }),
|
|
16
|
+
plugins: [new CamelCasePlugin(), new SerializePlugin()],
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
const TALLY = {
|
|
20
|
+
steps: 12,
|
|
21
|
+
calls: 30,
|
|
22
|
+
mutations: 4,
|
|
23
|
+
tokensIn: 900,
|
|
24
|
+
tokensOut: 120,
|
|
25
|
+
elapsed: 4200,
|
|
26
|
+
} as any
|
|
27
|
+
|
|
28
|
+
const FINDING = {
|
|
29
|
+
kind: 'schema-mismatch',
|
|
30
|
+
detail: 'listDocs returned a doc with no title',
|
|
31
|
+
rpcName: 'listDocs',
|
|
32
|
+
step: 7,
|
|
33
|
+
} as any
|
|
34
|
+
|
|
35
|
+
describe('KyselyVirtualUserRunStore', () => {
|
|
36
|
+
let db: Kysely<KyselyPikkuDB>
|
|
37
|
+
let store: KyselyVirtualUserRunStore
|
|
38
|
+
|
|
39
|
+
beforeEach(() => {
|
|
40
|
+
db = newDb()
|
|
41
|
+
store = new KyselyVirtualUserRunStore(db)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
// The runtime never needs this table, so it arrives with the feature that
|
|
45
|
+
// fills it rather than in every database — which means the first write has to
|
|
46
|
+
// create it.
|
|
47
|
+
test('creates its table on first use', async () => {
|
|
48
|
+
const runId = await store.start({
|
|
49
|
+
persona: 'susan',
|
|
50
|
+
disposition: 'realistic',
|
|
51
|
+
seed: 42,
|
|
52
|
+
})
|
|
53
|
+
assert.ok(runId)
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
test('a run starts as running, with no findings and no finish time', async () => {
|
|
57
|
+
const runId = await store.start({
|
|
58
|
+
persona: 'susan',
|
|
59
|
+
disposition: 'careless',
|
|
60
|
+
seed: 7,
|
|
61
|
+
goals: ['find the export button'],
|
|
62
|
+
memory: { docId: 'doc_1' },
|
|
63
|
+
startedBy: 'user_1',
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
const run = await store.get(runId)
|
|
67
|
+
assert.equal(run?.status, 'running')
|
|
68
|
+
assert.deepEqual(run?.findings, [])
|
|
69
|
+
assert.equal(run?.finishedAt, null)
|
|
70
|
+
assert.equal(run?.error, null)
|
|
71
|
+
assert.deepEqual(run?.goals, ['find the export button'])
|
|
72
|
+
assert.deepEqual(run?.memory, { docId: 'doc_1' })
|
|
73
|
+
assert.equal(run?.startedBy, 'user_1')
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
// The seed is the whole reason a finding is reproducible, and a BIGINT comes
|
|
77
|
+
// back as a string on some drivers — so it has to survive the round trip as a
|
|
78
|
+
// number.
|
|
79
|
+
test('the seed round-trips as a number', async () => {
|
|
80
|
+
const runId = await store.start({
|
|
81
|
+
persona: 'susan',
|
|
82
|
+
disposition: 'realistic',
|
|
83
|
+
seed: 2_147_483_646,
|
|
84
|
+
})
|
|
85
|
+
const run = await store.get(runId)
|
|
86
|
+
assert.strictEqual(run?.seed, 2_147_483_646)
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
test('completing a run stores what it found', async () => {
|
|
90
|
+
const runId = await store.start({
|
|
91
|
+
persona: 'susan',
|
|
92
|
+
disposition: 'auditor',
|
|
93
|
+
seed: 1,
|
|
94
|
+
memory: { docId: 'doc_1' },
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
await store.complete(runId, {
|
|
98
|
+
findings: [FINDING],
|
|
99
|
+
tally: TALLY,
|
|
100
|
+
memory: { docId: 'doc_1', orgId: 'org_9' },
|
|
101
|
+
stoppedBy: 'budget-steps',
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
const run = await store.get(runId)
|
|
105
|
+
assert.equal(run?.status, 'completed')
|
|
106
|
+
assert.equal(run?.findings.length, 1)
|
|
107
|
+
assert.equal(run?.findings[0]?.detail, FINDING.detail)
|
|
108
|
+
assert.deepEqual(run?.tally, TALLY)
|
|
109
|
+
assert.equal(run?.stoppedBy, 'budget-steps')
|
|
110
|
+
// Overwritten, not merged: the engine's memory already carries what it was
|
|
111
|
+
// given plus what it learned.
|
|
112
|
+
assert.deepEqual(run?.memory, { docId: 'doc_1', orgId: 'org_9' })
|
|
113
|
+
assert.ok(run?.finishedAt instanceof Date)
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
// A crashed run and a run that found nothing are different answers, and this
|
|
117
|
+
// record is the only place that distinction survives.
|
|
118
|
+
test('a failed run is failed, not an empty success', async () => {
|
|
119
|
+
const runId = await store.start({
|
|
120
|
+
persona: 'susan',
|
|
121
|
+
disposition: 'realistic',
|
|
122
|
+
seed: 1,
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
await store.fail(runId, 'SCENARIO_ACTOR_SECRET is not set')
|
|
126
|
+
|
|
127
|
+
const run = await store.get(runId)
|
|
128
|
+
assert.equal(run?.status, 'failed')
|
|
129
|
+
assert.equal(run?.error, 'SCENARIO_ACTOR_SECRET is not set')
|
|
130
|
+
assert.deepEqual(run?.findings, [])
|
|
131
|
+
assert.ok(run?.finishedAt instanceof Date)
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
test('an unknown run is null rather than a throw', async () => {
|
|
135
|
+
await store.init()
|
|
136
|
+
assert.equal(await store.get('nope'), null)
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
test('lists newest first', async () => {
|
|
140
|
+
const first = await store.start({
|
|
141
|
+
persona: 'susan',
|
|
142
|
+
disposition: 'realistic',
|
|
143
|
+
seed: 1,
|
|
144
|
+
})
|
|
145
|
+
const second = await store.start({
|
|
146
|
+
persona: 'raj',
|
|
147
|
+
disposition: 'realistic',
|
|
148
|
+
seed: 2,
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
const runs = await store.list()
|
|
152
|
+
assert.equal(runs.length, 2)
|
|
153
|
+
assert.deepEqual(
|
|
154
|
+
new Set(runs.map((run) => run.runId)),
|
|
155
|
+
new Set([first, second])
|
|
156
|
+
)
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
// Comparing what a persona finds this week against last week is the whole
|
|
160
|
+
// reason runs are kept.
|
|
161
|
+
test('narrows to one persona s history', async () => {
|
|
162
|
+
await store.start({ persona: 'susan', disposition: 'realistic', seed: 1 })
|
|
163
|
+
await store.start({ persona: 'raj', disposition: 'realistic', seed: 2 })
|
|
164
|
+
|
|
165
|
+
const runs = await store.list({ persona: 'susan' })
|
|
166
|
+
assert.equal(runs.length, 1)
|
|
167
|
+
assert.equal(runs[0]?.persona, 'susan')
|
|
168
|
+
})
|
|
169
|
+
|
|
170
|
+
test('completing an unknown run does not throw', async () => {
|
|
171
|
+
await store.init()
|
|
172
|
+
await store.complete('nope', {
|
|
173
|
+
findings: [],
|
|
174
|
+
tally: TALLY,
|
|
175
|
+
memory: {},
|
|
176
|
+
stoppedBy: 'exhausted',
|
|
177
|
+
})
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
// Not every project installs SerializePlugin, and a bare SQLite driver cannot
|
|
181
|
+
// bind a Date at all — so the whole round trip has to hold without it.
|
|
182
|
+
test('round-trips without SerializePlugin', async () => {
|
|
183
|
+
const bare = new Kysely<KyselyPikkuDB>({
|
|
184
|
+
dialect: new SqliteDialect({ database: new Database(':memory:') }),
|
|
185
|
+
plugins: [new CamelCasePlugin()],
|
|
186
|
+
})
|
|
187
|
+
const bareStore = new KyselyVirtualUserRunStore(bare)
|
|
188
|
+
|
|
189
|
+
const runId = await bareStore.start({
|
|
190
|
+
persona: 'susan',
|
|
191
|
+
disposition: 'realistic',
|
|
192
|
+
seed: 99,
|
|
193
|
+
goals: ['export a doc'],
|
|
194
|
+
memory: { docId: 'doc_1' },
|
|
195
|
+
})
|
|
196
|
+
await bareStore.complete(runId, {
|
|
197
|
+
findings: [FINDING],
|
|
198
|
+
tally: TALLY,
|
|
199
|
+
memory: { docId: 'doc_1' },
|
|
200
|
+
stoppedBy: 'exhausted',
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
const run = await bareStore.get(runId)
|
|
204
|
+
assert.equal(run?.status, 'completed')
|
|
205
|
+
assert.strictEqual(run?.seed, 99)
|
|
206
|
+
assert.deepEqual(run?.goals, ['export a doc'])
|
|
207
|
+
assert.deepEqual(run?.tally, TALLY)
|
|
208
|
+
assert.equal(run?.findings[0]?.detail, FINDING.detail)
|
|
209
|
+
assert.ok(run?.createdAt instanceof Date)
|
|
210
|
+
assert.ok(run?.finishedAt instanceof Date)
|
|
211
|
+
})
|
|
212
|
+
})
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import type { Kysely, Selectable } from 'kysely'
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
3
|
+
import type {
|
|
4
|
+
VirtualUserDisposition,
|
|
5
|
+
VirtualUserRunOutcome,
|
|
6
|
+
VirtualUserRunRecord,
|
|
7
|
+
VirtualUserRunStart,
|
|
8
|
+
VirtualUserRunStore,
|
|
9
|
+
} from '@pikku/core/virtual-user'
|
|
10
|
+
import type { KyselyPikkuDB, VirtualUserRunTable } from './kysely-tables.js'
|
|
11
|
+
import { parseJson } from './kysely-json.js'
|
|
12
|
+
import { ensurePikkuSchema } from './schema/index.js'
|
|
13
|
+
import { virtualUserSchema } from './schema/virtual-user.schema.js'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Records virtual-user runs in a `virtualUserRun` table.
|
|
17
|
+
*
|
|
18
|
+
* The store is the whole record of a run: `runVirtualUser` returns as soon as
|
|
19
|
+
* the row exists and the engine keeps going without it, so nothing else knows
|
|
20
|
+
* the run happened. That is why `fail()` exists as its own call — a run that
|
|
21
|
+
* crashed and a run that found nothing are different answers, and a row left at
|
|
22
|
+
* `running` is neither.
|
|
23
|
+
*
|
|
24
|
+
* Works with or without `SerializePlugin`, which not every project installs: the
|
|
25
|
+
* JSON columns are written as strings (which the plugin passes through) and read
|
|
26
|
+
* back through `parseJson` (which passes through what the plugin already
|
|
27
|
+
* parsed). Timestamps are written as ISO strings for the same reason — a bare
|
|
28
|
+
* SQLite driver cannot bind a `Date` at all.
|
|
29
|
+
*
|
|
30
|
+
* SECURITY: an `adversarial` run's findings are working exploits carrying live
|
|
31
|
+
* ids. This is a privileged store; the scaffolded reads are scope-gated, and
|
|
32
|
+
* exposing these rows more widely publishes your own exploits.
|
|
33
|
+
*/
|
|
34
|
+
export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
35
|
+
private initialized = false
|
|
36
|
+
|
|
37
|
+
constructor(private db: Kysely<KyselyPikkuDB>) {}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Creates the table on first use, like the audit sink — the runtime does not
|
|
41
|
+
* need it, so it arrives with the feature that fills it rather than in every
|
|
42
|
+
* database.
|
|
43
|
+
*/
|
|
44
|
+
public async init(): Promise<void> {
|
|
45
|
+
if (this.initialized) return
|
|
46
|
+
await ensurePikkuSchema(this.db, virtualUserSchema)
|
|
47
|
+
this.initialized = true
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async start(run: VirtualUserRunStart): Promise<string> {
|
|
51
|
+
await this.init()
|
|
52
|
+
const runId = randomUUID()
|
|
53
|
+
await this.db
|
|
54
|
+
.insertInto('virtualUserRun')
|
|
55
|
+
.values({
|
|
56
|
+
runId,
|
|
57
|
+
persona: run.persona,
|
|
58
|
+
disposition: run.disposition,
|
|
59
|
+
// Text on every engine (see the schema), so the seed round-trips
|
|
60
|
+
// through drivers that hand a BIGINT back as a string.
|
|
61
|
+
seed: String(run.seed),
|
|
62
|
+
status: 'running',
|
|
63
|
+
goals: JSON.stringify(run.goals ?? []),
|
|
64
|
+
memory: JSON.stringify(run.memory ?? {}),
|
|
65
|
+
findings: '[]',
|
|
66
|
+
startedBy: run.startedBy ?? null,
|
|
67
|
+
})
|
|
68
|
+
.execute()
|
|
69
|
+
return runId
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void> {
|
|
73
|
+
await this.init()
|
|
74
|
+
await this.db
|
|
75
|
+
.updateTable('virtualUserRun')
|
|
76
|
+
.set({
|
|
77
|
+
status: 'completed',
|
|
78
|
+
findings: JSON.stringify(outcome.findings),
|
|
79
|
+
tally: JSON.stringify(outcome.tally),
|
|
80
|
+
// Overwritten rather than merged: the engine's memory already carries
|
|
81
|
+
// what it was given, plus what it learned on the way.
|
|
82
|
+
memory: JSON.stringify(outcome.memory),
|
|
83
|
+
stoppedBy: outcome.stoppedBy,
|
|
84
|
+
finishedAt: new Date().toISOString(),
|
|
85
|
+
})
|
|
86
|
+
.where('runId', '=', runId)
|
|
87
|
+
.execute()
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async fail(runId: string, error: string): Promise<void> {
|
|
91
|
+
await this.init()
|
|
92
|
+
await this.db
|
|
93
|
+
.updateTable('virtualUserRun')
|
|
94
|
+
.set({
|
|
95
|
+
status: 'failed',
|
|
96
|
+
error,
|
|
97
|
+
finishedAt: new Date().toISOString(),
|
|
98
|
+
})
|
|
99
|
+
.where('runId', '=', runId)
|
|
100
|
+
.execute()
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async get(runId: string): Promise<VirtualUserRunRecord | null> {
|
|
104
|
+
await this.init()
|
|
105
|
+
const row = await this.db
|
|
106
|
+
.selectFrom('virtualUserRun')
|
|
107
|
+
.selectAll()
|
|
108
|
+
.where('runId', '=', runId)
|
|
109
|
+
.executeTakeFirst()
|
|
110
|
+
return row ? this.toRecord(row) : null
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async list(options?: {
|
|
114
|
+
persona?: string
|
|
115
|
+
limit?: number
|
|
116
|
+
offset?: number
|
|
117
|
+
}): Promise<VirtualUserRunRecord[]> {
|
|
118
|
+
await this.init()
|
|
119
|
+
let query = this.db.selectFrom('virtualUserRun').selectAll()
|
|
120
|
+
if (options?.persona) {
|
|
121
|
+
query = query.where('persona', '=', options.persona)
|
|
122
|
+
}
|
|
123
|
+
const rows = await query
|
|
124
|
+
.orderBy('createdAt', 'desc')
|
|
125
|
+
.limit(options?.limit ?? 50)
|
|
126
|
+
.offset(options?.offset ?? 0)
|
|
127
|
+
.execute()
|
|
128
|
+
return rows.map((row) => this.toRecord(row))
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private toRecord(row: Selectable<VirtualUserRunTable>): VirtualUserRunRecord {
|
|
132
|
+
return {
|
|
133
|
+
runId: row.runId,
|
|
134
|
+
persona: row.persona,
|
|
135
|
+
disposition: row.disposition as VirtualUserDisposition,
|
|
136
|
+
seed: Number(row.seed),
|
|
137
|
+
status: row.status,
|
|
138
|
+
goals: parseJson(row.goals) ?? [],
|
|
139
|
+
memory: parseJson(row.memory) ?? {},
|
|
140
|
+
findings: parseJson(row.findings) ?? [],
|
|
141
|
+
tally: row.tally ? (parseJson(row.tally) ?? null) : null,
|
|
142
|
+
stoppedBy: row.stoppedBy,
|
|
143
|
+
error: row.error,
|
|
144
|
+
startedBy: row.startedBy,
|
|
145
|
+
createdAt: new Date(row.createdAt),
|
|
146
|
+
finishedAt: row.finishedAt ? new Date(row.finishedAt) : null,
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -505,23 +505,6 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
|
|
|
505
505
|
return { completedNodeIds, failedNodeIds, branchKeys }
|
|
506
506
|
}
|
|
507
507
|
|
|
508
|
-
async getNodesWithoutSteps(
|
|
509
|
-
runId: string,
|
|
510
|
-
nodeIds: string[]
|
|
511
|
-
): Promise<string[]> {
|
|
512
|
-
if (nodeIds.length === 0) return []
|
|
513
|
-
|
|
514
|
-
const result = await this.db
|
|
515
|
-
.selectFrom('workflowStep')
|
|
516
|
-
.select('stepName')
|
|
517
|
-
.where('workflowRunId', '=', runId)
|
|
518
|
-
.where('stepName', 'in', nodeIds)
|
|
519
|
-
.execute()
|
|
520
|
-
|
|
521
|
-
const existingStepNames = new Set(result.map((r) => r.stepName))
|
|
522
|
-
return nodeIds.filter((id) => !existingStepNames.has(id))
|
|
523
|
-
}
|
|
524
|
-
|
|
525
508
|
async getStepInstances(
|
|
526
509
|
runId: string
|
|
527
510
|
): Promise<
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { PikkuSchema } from './pikku-schema.types.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The `audit` table {@link KyselyAuditService} writes to and reads back.
|
|
5
|
+
*
|
|
6
|
+
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
7
|
+
* a project that has wired a durable audit sink does, and creating it for
|
|
8
|
+
* everyone would put an empty table in every database. `KyselyAuditService.init()`
|
|
9
|
+
* applies it, so it arrives exactly when the sink that fills it does.
|
|
10
|
+
*
|
|
11
|
+
* Every column is text on every engine, matching the platform audit-queue
|
|
12
|
+
* consumer's row shape — a locally-run project and a deployed stage write rows
|
|
13
|
+
* the same reader can read. `occurredAt` is an ISO 8601 string rather than a
|
|
14
|
+
* timestamp for that reason: string ordering is chronological ordering, and the
|
|
15
|
+
* queue consumer has no shared type to agree on.
|
|
16
|
+
*/
|
|
17
|
+
export const auditSchema: PikkuSchema = {
|
|
18
|
+
name: 'audit',
|
|
19
|
+
statements: [
|
|
20
|
+
(db) =>
|
|
21
|
+
db.schema
|
|
22
|
+
.createTable('audit')
|
|
23
|
+
.addColumn('auditId', 'text', (col) => col.primaryKey())
|
|
24
|
+
.addColumn('occurredAt', 'text', (col) => col.notNull())
|
|
25
|
+
.addColumn('type', 'text', (col) => col.notNull())
|
|
26
|
+
.addColumn('source', 'text', (col) => col.defaultTo('auto').notNull())
|
|
27
|
+
.addColumn('outcome', 'text')
|
|
28
|
+
.addColumn('functionId', 'text')
|
|
29
|
+
.addColumn('wireType', 'text')
|
|
30
|
+
.addColumn('traceId', 'text')
|
|
31
|
+
.addColumn('transactionId', 'text')
|
|
32
|
+
.addColumn('queryId', 'text')
|
|
33
|
+
.addColumn('userId', 'text')
|
|
34
|
+
.addColumn('orgId', 'text')
|
|
35
|
+
.addColumn('pikkuUserId', 'text')
|
|
36
|
+
.addColumn('tables', 'text')
|
|
37
|
+
.addColumn('changedCols', 'text')
|
|
38
|
+
.addColumn('event', 'text')
|
|
39
|
+
.addColumn('old', 'text')
|
|
40
|
+
.addColumn('data', 'text'),
|
|
41
|
+
|
|
42
|
+
// The trail is only ever read newest-first, and the two filters the console
|
|
43
|
+
// offers are user and type.
|
|
44
|
+
(db) =>
|
|
45
|
+
db.schema
|
|
46
|
+
.createIndex('idx_audit_occurred_at')
|
|
47
|
+
.on('audit')
|
|
48
|
+
.column('occurredAt'),
|
|
49
|
+
|
|
50
|
+
(db) =>
|
|
51
|
+
db.schema.createIndex('idx_audit_user_id').on('audit').column('userId'),
|
|
52
|
+
|
|
53
|
+
(db) => db.schema.createIndex('idx_audit_type').on('audit').column('type'),
|
|
54
|
+
],
|
|
55
|
+
}
|
|
@@ -7,8 +7,9 @@ import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
|
|
|
7
7
|
* The uniqueness rule is an expression index — `user_id` is nullable and two
|
|
8
8
|
* rows with a null user must still collide on name — which the schema builder
|
|
9
9
|
* cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
|
|
10
|
-
* which is why the table
|
|
11
|
-
* path.
|
|
10
|
+
* which is why the table goes through `ctx.table` rather than being left to the
|
|
11
|
+
* search path. The index's own name stays unqualified: an index belongs to the
|
|
12
|
+
* schema of the table it is on, and postgres rejects one that says otherwise.
|
|
12
13
|
*/
|
|
13
14
|
export const credentialSchema: PikkuSchema = {
|
|
14
15
|
name: 'credential',
|
|
@@ -29,9 +30,19 @@ export const credentialSchema: PikkuSchema = {
|
|
|
29
30
|
),
|
|
30
31
|
|
|
31
32
|
rawStatement(
|
|
32
|
-
|
|
33
|
+
({ table }) =>
|
|
34
|
+
sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON ${table('credentials')} (name, COALESCE(user_id, ''))`
|
|
33
35
|
),
|
|
34
36
|
|
|
37
|
+
(db) =>
|
|
38
|
+
db.schema
|
|
39
|
+
.createTable('credentialKekSalts')
|
|
40
|
+
.addColumn('keyVersion', 'integer', (col) => col.primaryKey())
|
|
41
|
+
.addColumn('salt', 'varchar(64)', (col) => col.notNull())
|
|
42
|
+
.addColumn('createdAt', 'timestamp', (col) =>
|
|
43
|
+
col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
|
|
44
|
+
),
|
|
45
|
+
|
|
35
46
|
(db) =>
|
|
36
47
|
db.schema
|
|
37
48
|
.createTable('credentialsAudit')
|
package/src/schema/index.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { CamelCasePlugin, type Kysely } from 'kysely'
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import {
|
|
3
|
+
schemaContext,
|
|
4
|
+
type PikkuSchema,
|
|
5
|
+
type RequiredTypes,
|
|
6
|
+
type SchemaContext,
|
|
7
|
+
type SchemaRequirement,
|
|
6
8
|
} from './pikku-schema.types.js'
|
|
7
9
|
import { aiSchema } from './ai.schema.js'
|
|
8
10
|
import { channelSchema } from './channel.schema.js'
|
|
@@ -17,11 +19,24 @@ import { workflowSchema } from './workflow.schema.js'
|
|
|
17
19
|
export type {
|
|
18
20
|
PikkuSchema,
|
|
19
21
|
RequiredTypes,
|
|
22
|
+
SchemaContext,
|
|
20
23
|
SchemaRequirement,
|
|
21
24
|
SchemaStatement,
|
|
22
25
|
SchemaStatementFactory,
|
|
23
26
|
} from './pikku-schema.types.js'
|
|
24
|
-
export {
|
|
27
|
+
export {
|
|
28
|
+
rawStatement,
|
|
29
|
+
requiredType,
|
|
30
|
+
schemaContext,
|
|
31
|
+
unqualifiedContext,
|
|
32
|
+
} from './pikku-schema.types.js'
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
|
|
36
|
+
* runtime works without an audit table, and `KyselyAuditService.init()` applies
|
|
37
|
+
* this for the projects that wire the sink.
|
|
38
|
+
*/
|
|
39
|
+
export { auditSchema } from './audit.schema.js'
|
|
25
40
|
|
|
26
41
|
/**
|
|
27
42
|
* Every table the pikku runtime needs, in dependency order.
|
|
@@ -53,6 +68,29 @@ export const pikkuSchemas: PikkuSchema[] = [
|
|
|
53
68
|
*/
|
|
54
69
|
const bind = (db: Kysely<any>) => db.withPlugin(new CamelCasePlugin())
|
|
55
70
|
|
|
71
|
+
/**
|
|
72
|
+
* The schema a connection is bound to, or `undefined` for one that is not.
|
|
73
|
+
*
|
|
74
|
+
* Read back out of compiled DDL rather than off the connection, because kysely
|
|
75
|
+
* keeps the `withSchema(...)` setting inside its executor and exposes no way to
|
|
76
|
+
* ask. Compiling a throwaway `create table` is the same trick `declaredTables`
|
|
77
|
+
* uses on the real statements, against the same regex — nothing is executed and
|
|
78
|
+
* the probe never leaves this function.
|
|
79
|
+
*
|
|
80
|
+
* Derived rather than passed in so that raw statements follow whatever
|
|
81
|
+
* connection they are handed. A caller that had to remember to say the schema
|
|
82
|
+
* twice would eventually say it once.
|
|
83
|
+
*/
|
|
84
|
+
const boundSchema = (db: Kysely<any>): string | undefined =>
|
|
85
|
+
CREATE_TABLE.exec(
|
|
86
|
+
db.schema.createTable('pikkuSchemaProbe').addColumn('id', 'text').compile()
|
|
87
|
+
.sql
|
|
88
|
+
)?.[1]
|
|
89
|
+
|
|
90
|
+
/** The context for statements compiled against `db`, matching its binding. */
|
|
91
|
+
const contextFor = (db: Kysely<any>): SchemaContext =>
|
|
92
|
+
schemaContext(boundSchema(db))
|
|
93
|
+
|
|
56
94
|
export interface UnmetRequirement {
|
|
57
95
|
schema: PikkuSchema
|
|
58
96
|
requirement: SchemaRequirement
|
|
@@ -141,9 +179,10 @@ export const applyPikkuSchemas = async (
|
|
|
141
179
|
|
|
142
180
|
await db.transaction().execute(async (trx) => {
|
|
143
181
|
const bound = bind(trx)
|
|
182
|
+
const ctx = contextFor(bound)
|
|
144
183
|
for (const schema of schemas) {
|
|
145
184
|
for (const statement of schema.statements) {
|
|
146
|
-
const query = statement(bound, types)
|
|
185
|
+
const query = statement(bound, types, ctx)
|
|
147
186
|
try {
|
|
148
187
|
await query.execute()
|
|
149
188
|
} catch (error) {
|
|
@@ -239,9 +278,10 @@ export const declaredTables = (
|
|
|
239
278
|
db: Kysely<any>
|
|
240
279
|
): DeclaredTable[] => {
|
|
241
280
|
const bound = bind(db)
|
|
281
|
+
const ctx = contextFor(bound)
|
|
242
282
|
const tables: DeclaredTable[] = []
|
|
243
283
|
for (const statement of schema.statements) {
|
|
244
|
-
const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql)
|
|
284
|
+
const match = CREATE_TABLE.exec(statement(bound, {}, ctx).compile().sql)
|
|
245
285
|
if (match) tables.push({ schema: match[1], name: match[2]! })
|
|
246
286
|
}
|
|
247
287
|
return tables
|
|
@@ -357,17 +397,26 @@ export const ensurePikkuSchema = async (
|
|
|
357
397
|
* `types` comes from `resolveRequirements` against the database the migration
|
|
358
398
|
* is being written for. Omit it and foreign keys onto another source's columns
|
|
359
399
|
* compile as `text`, which is a shape, not a migration.
|
|
400
|
+
*
|
|
401
|
+
* `schema` qualifies every table the migration creates, for a project that
|
|
402
|
+
* keeps the runtime tables somewhere other than the default search path. It is
|
|
403
|
+
* applied here and not to `db` itself because the caller's connection is a
|
|
404
|
+
* throwaway the declaration was just *applied* to, in whatever schema that
|
|
405
|
+
* database actually has — qualifying that would create tables in a schema the
|
|
406
|
+
* scratch database has never heard of. Only the rendered SQL is bound.
|
|
360
407
|
*/
|
|
361
408
|
export const compilePikkuSchemas = (
|
|
362
409
|
db: Kysely<any>,
|
|
363
410
|
schemas: PikkuSchema[] = pikkuSchemas,
|
|
364
|
-
types: RequiredTypes = {}
|
|
411
|
+
types: RequiredTypes = {},
|
|
412
|
+
schemaName?: string
|
|
365
413
|
): string => {
|
|
366
|
-
const bound = bind(db)
|
|
414
|
+
const bound = schemaName ? bind(db).withSchema(schemaName) : bind(db)
|
|
415
|
+
const ctx = contextFor(bound)
|
|
367
416
|
return schemas
|
|
368
417
|
.flatMap((schema) =>
|
|
369
418
|
schema.statements.map(
|
|
370
|
-
(statement) => `${statement(bound, types).compile().sql};`
|
|
419
|
+
(statement) => `${statement(bound, types, ctx).compile().sql};`
|
|
371
420
|
)
|
|
372
421
|
)
|
|
373
422
|
.join('\n\n')
|
|
@@ -15,9 +15,51 @@ export interface SchemaStatement {
|
|
|
15
15
|
|
|
16
16
|
export type SchemaStatementFactory = (
|
|
17
17
|
db: Kysely<any>,
|
|
18
|
-
types: RequiredTypes
|
|
18
|
+
types: RequiredTypes,
|
|
19
|
+
ctx: SchemaContext
|
|
19
20
|
) => SchemaStatement
|
|
20
21
|
|
|
22
|
+
/**
|
|
23
|
+
* What a statement needs to know about the schema its DDL lands in.
|
|
24
|
+
*
|
|
25
|
+
* Only raw SQL needs this. The schema builder learns the schema from the
|
|
26
|
+
* connection — `withSchema('app')` qualifies every table it names, foreign key
|
|
27
|
+
* targets included — but a raw fragment is passed through verbatim, so a
|
|
28
|
+
* statement that spells its own SQL has to spell the qualifier too.
|
|
29
|
+
*/
|
|
30
|
+
export interface SchemaContext {
|
|
31
|
+
/**
|
|
32
|
+
* A table as the surrounding DDL addresses it: `"app"."credentials"` on a
|
|
33
|
+
* schema-bound connection, `"credentials"` otherwise.
|
|
34
|
+
*
|
|
35
|
+
* Takes the physical name, not the camelCase one the schema builder accepts.
|
|
36
|
+
* Raw SQL does not go through `CamelCasePlugin`, so it is already written in
|
|
37
|
+
* physical names throughout and this is no exception.
|
|
38
|
+
*/
|
|
39
|
+
table: (name: string) => RawBuilder<unknown>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The `SchemaContext` for DDL that names no schema.
|
|
44
|
+
*
|
|
45
|
+
* The default everywhere a schema has not been asked for, which is every engine
|
|
46
|
+
* but postgres and most postgres projects too.
|
|
47
|
+
*/
|
|
48
|
+
export const unqualifiedContext: SchemaContext = {
|
|
49
|
+
table: (name) => sql.raw(`"${name}"`),
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A `SchemaContext` qualifying every table into `schema`.
|
|
54
|
+
*
|
|
55
|
+
* Identifiers are quoted rather than interpolated bare so a schema named after
|
|
56
|
+
* a reserved word still compiles.
|
|
57
|
+
*/
|
|
58
|
+
export const schemaContext = (schema: string | undefined): SchemaContext =>
|
|
59
|
+
schema
|
|
60
|
+
? { table: (name) => sql.raw(`"${schema}"."${name}"`) }
|
|
61
|
+
: unqualifiedContext
|
|
62
|
+
|
|
21
63
|
/**
|
|
22
64
|
* The physical types of the columns named in `requires`, keyed `table.column`.
|
|
23
65
|
*
|
|
@@ -82,13 +124,26 @@ export interface SchemaRequirement {
|
|
|
82
124
|
* Wrap a raw SQL fragment as a statement.
|
|
83
125
|
*
|
|
84
126
|
* For the DDL kysely's schema builder cannot express — a unique index over an
|
|
85
|
-
* expression, say.
|
|
86
|
-
*
|
|
87
|
-
*
|
|
127
|
+
* expression, say. Raw SQL is *not* rewritten by `withSchema`, so a fragment
|
|
128
|
+
* naming a table unqualified resolves against the connection's search path
|
|
129
|
+
* rather than the schema the rest of the declaration lands in. Take the
|
|
130
|
+
* function form and name tables through `ctx.table` to stay with them; the
|
|
131
|
+
* bare-fragment form is for SQL that names no table at all.
|
|
88
132
|
*/
|
|
89
|
-
export
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
133
|
+
export function rawStatement(
|
|
134
|
+
fragment: RawBuilder<unknown>
|
|
135
|
+
): SchemaStatementFactory
|
|
136
|
+
export function rawStatement(
|
|
137
|
+
build: (ctx: SchemaContext) => RawBuilder<unknown>
|
|
138
|
+
): SchemaStatementFactory
|
|
139
|
+
export function rawStatement(
|
|
140
|
+
source: RawBuilder<unknown> | ((ctx: SchemaContext) => RawBuilder<unknown>)
|
|
141
|
+
): SchemaStatementFactory {
|
|
142
|
+
return (db, _types, ctx = unqualifiedContext) => {
|
|
143
|
+
const fragment = typeof source === 'function' ? source(ctx) : source
|
|
144
|
+
return {
|
|
145
|
+
execute: () => fragment.execute(db),
|
|
146
|
+
compile: () => fragment.compile(db),
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|