@pikku/kysely 0.13.20 → 0.13.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -0
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.js +1 -0
- package/dist/src/kysely-tables.d.ts +46 -0
- package/dist/src/kysely-virtual-user-run-store.d.ts +7 -19
- package/dist/src/kysely-virtual-user-run-store.js +69 -2
- package/dist/src/kysely-virtual-user-schedule-store.d.ts +33 -0
- package/dist/src/kysely-virtual-user-schedule-store.js +151 -0
- package/dist/src/schema/index.d.ts +1 -0
- package/dist/src/schema/index.js +3 -0
- package/dist/src/schema/virtual-user-schedule.schema.d.ts +15 -0
- package/dist/src/schema/virtual-user-schedule.schema.js +43 -0
- package/dist/src/schema/virtual-user.schema.d.ts +2 -1
- package/dist/src/schema/virtual-user.schema.js +39 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/index.ts +3 -0
- package/src/kysely-tables.ts +48 -0
- package/src/kysely-virtual-user-run-store.test.ts +142 -1
- package/src/kysely-virtual-user-run-store.ts +90 -4
- package/src/kysely-virtual-user-schedule-store.test.ts +187 -0
- package/src/kysely-virtual-user-schedule-store.ts +187 -0
- package/src/kysely-workflow-service.test.ts +1 -1
- package/src/schema/index.ts +3 -0
- package/src/schema/virtual-user-schedule.schema.ts +48 -0
- package/src/schema/virtual-user.schema.ts +41 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { sql } from 'kysely';
|
|
2
2
|
/**
|
|
3
|
-
* The `virtualUserRun`
|
|
3
|
+
* The `virtualUserRun` and `virtualUserRunStep` tables
|
|
4
|
+
* {@link KyselyVirtualUserRunStore} writes to.
|
|
4
5
|
*
|
|
5
6
|
* Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
|
|
6
7
|
* a project that runs virtual users does, and creating it for everyone would
|
|
@@ -33,6 +34,10 @@ export const virtualUserSchema = {
|
|
|
33
34
|
.addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
|
|
34
35
|
.addColumn('memory', 'text', (col) => col.defaultTo('{}').notNull())
|
|
35
36
|
.addColumn('findings', 'text', (col) => col.defaultTo('[]').notNull())
|
|
37
|
+
// What the user set out to do and how far each one got. On the run row
|
|
38
|
+
// rather than beside the steps: there are a handful of them, and every
|
|
39
|
+
// read of the run wants them.
|
|
40
|
+
.addColumn('intents', 'text', (col) => col.defaultTo('[]').notNull())
|
|
36
41
|
// Counts the engine kept: steps, calls, mutations, tokens, elapsed.
|
|
37
42
|
.addColumn('tally', 'text')
|
|
38
43
|
.addColumn('stoppedBy', 'varchar(255)')
|
|
@@ -51,5 +56,38 @@ export const virtualUserSchema = {
|
|
|
51
56
|
.createIndex('idx_virtual_user_run_persona')
|
|
52
57
|
.on('virtualUserRun')
|
|
53
58
|
.columns(['persona', 'createdAt']),
|
|
59
|
+
// One row per turn, kept out of `virtualUserRun` so that listing runs does
|
|
60
|
+
// not drag a budget's worth of transcript along with it. No foreign key
|
|
61
|
+
// onto the run for the same reason `persona` is not one: this table is
|
|
62
|
+
// created by the store at boot on whatever engine the project uses, and a
|
|
63
|
+
// cascade that half the engines enforce differently is worse than the read
|
|
64
|
+
// rule that a step with no run is orphaned.
|
|
65
|
+
(db) => db.schema
|
|
66
|
+
.createTable('virtualUserRunStep')
|
|
67
|
+
.addColumn('runId', 'varchar(36)', (col) => col.notNull())
|
|
68
|
+
// The engine's own step number, which is what the intent records point
|
|
69
|
+
// at — not a surrogate key, so a transcript reads the same after paging.
|
|
70
|
+
// Spelled `stepIndex` because `index` is a reserved word in mysql.
|
|
71
|
+
.addColumn('stepIndex', 'integer', (col) => col.notNull())
|
|
72
|
+
.addColumn('intentId', 'varchar(255)')
|
|
73
|
+
// The action as the engine scheduled it, including the `invalid` shape
|
|
74
|
+
// for a turn the model got wrong — that turn is the interesting one.
|
|
75
|
+
.addColumn('action', 'text', (col) => col.notNull())
|
|
76
|
+
.addColumn('status', 'integer')
|
|
77
|
+
// 0 or 1 rather than a boolean, because a bare sqlite driver cannot
|
|
78
|
+
// bind one at all and `SerializePlugin` is not installed everywhere.
|
|
79
|
+
.addColumn('ok', 'integer')
|
|
80
|
+
// Already truncated by the engine, and stored JSON-encoded rather than
|
|
81
|
+
// raw: a truncated API response usually starts with a brace, which
|
|
82
|
+
// `SerializePlugin` would read back as an object rather than the string
|
|
83
|
+
// the engine actually saw.
|
|
84
|
+
.addColumn('response', 'text')
|
|
85
|
+
.addColumn('findingKinds', 'text')
|
|
86
|
+
.addColumn('tokensIn', 'integer', (col) => col.defaultTo(0).notNull())
|
|
87
|
+
.addColumn('tokensOut', 'integer', (col) => col.defaultTo(0).notNull())
|
|
88
|
+
.addPrimaryKeyConstraint('pk_virtual_user_run_step', [
|
|
89
|
+
'runId',
|
|
90
|
+
'stepIndex',
|
|
91
|
+
]),
|
|
54
92
|
],
|
|
55
93
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"root":["../src/coercion-plugin.ts","../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-agent-run-scores.ts","../src/kysely-agent-run-service.ts","../src/kysely-agent-run-state-service.ts","../src/kysely-agent-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-virtual-user-run-store.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/agent.schema.ts","../src/schema/audit.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/virtual-user.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}
|
|
1
|
+
{"root":["../src/coercion-plugin.ts","../src/create-audited-kysely.ts","../src/index.ts","../src/kysely-agent-run-scores.ts","../src/kysely-agent-run-service.ts","../src/kysely-agent-run-state-service.ts","../src/kysely-agent-storage-service.ts","../src/kysely-audit-service.ts","../src/kysely-channel-store.ts","../src/kysely-credential-service.ts","../src/kysely-deployment-service.ts","../src/kysely-eventhub-store.ts","../src/kysely-json.ts","../src/kysely-scope-service.ts","../src/kysely-secret-service.ts","../src/kysely-session-store.ts","../src/kysely-tables.ts","../src/kysely-virtual-user-run-store.ts","../src/kysely-virtual-user-schedule-store.ts","../src/kysely-webhook-service.ts","../src/kysely-workflow-mirror.ts","../src/kysely-workflow-run-service.ts","../src/kysely-workflow-service.ts","../src/serialize-plugin.ts","../src/schema/agent.schema.ts","../src/schema/audit.schema.ts","../src/schema/channel.schema.ts","../src/schema/credential.schema.ts","../src/schema/deployment.schema.ts","../src/schema/index.ts","../src/schema/pikku-schema.types.ts","../src/schema/scope.schema.ts","../src/schema/secret.schema.ts","../src/schema/session.schema.ts","../src/schema/virtual-user-schedule.schema.ts","../src/schema/virtual-user.schema.ts","../src/schema/webhook.schema.ts","../src/schema/workflow.schema.ts","../bin/pikku-kysely-pure.ts"],"version":"6.0.3"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/kysely",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.21",
|
|
4
4
|
"description": "Kysely-backed Pikku services — workflow state, secrets, sessions, scopes and audit",
|
|
5
5
|
"author": "yasser.fadl@gmail.com",
|
|
6
6
|
"license": "MIT",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"prepublishOnly": "yarn build"
|
|
22
22
|
},
|
|
23
23
|
"peerDependencies": {
|
|
24
|
-
"@pikku/core": "^0.12.
|
|
24
|
+
"@pikku/core": "^0.12.90"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"kysely": "^0.29.5"
|
package/src/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ export {
|
|
|
18
18
|
} from './create-audited-kysely.js'
|
|
19
19
|
export { KyselyAuditService } from './kysely-audit-service.js'
|
|
20
20
|
export { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
|
|
21
|
+
export { KyselyVirtualUserScheduleStore } from './kysely-virtual-user-schedule-store.js'
|
|
21
22
|
|
|
22
23
|
export {
|
|
23
24
|
SerializePlugin,
|
|
@@ -61,4 +62,6 @@ export type { AgentRunService, AgentRunRow } from '@pikku/core/agent'
|
|
|
61
62
|
export type {
|
|
62
63
|
VirtualUserRunStore,
|
|
63
64
|
VirtualUserRunRecord,
|
|
65
|
+
VirtualUserScheduleStore,
|
|
66
|
+
VirtualUserScheduleRecord,
|
|
64
67
|
} from '@pikku/core/virtual-user'
|
package/src/kysely-tables.ts
CHANGED
|
@@ -311,6 +311,7 @@ export interface VirtualUserRunTable {
|
|
|
311
311
|
goals: Generated<string>
|
|
312
312
|
memory: Generated<string>
|
|
313
313
|
findings: Generated<string>
|
|
314
|
+
intents: Generated<string>
|
|
314
315
|
tally: string | null
|
|
315
316
|
stoppedBy: string | null
|
|
316
317
|
error: string | null
|
|
@@ -352,6 +353,51 @@ export interface AuditTable {
|
|
|
352
353
|
data: string | null
|
|
353
354
|
}
|
|
354
355
|
|
|
356
|
+
/**
|
|
357
|
+
* One turn of a run's transcript.
|
|
358
|
+
*
|
|
359
|
+
* Its own table rather than a column on {@link VirtualUserRunTable}: a run at a
|
|
360
|
+
* 500-step budget carries more transcript than every other column together, and
|
|
361
|
+
* the run list would pay for it on every row.
|
|
362
|
+
*/
|
|
363
|
+
export interface VirtualUserRunStepTable {
|
|
364
|
+
runId: string
|
|
365
|
+
/** The engine's step number. `stepIndex` because `index` is reserved in mysql. */
|
|
366
|
+
stepIndex: number
|
|
367
|
+
intentId: string | null
|
|
368
|
+
/** JSON: the action the engine scheduled, or the `invalid` shape. */
|
|
369
|
+
action: string
|
|
370
|
+
status: number | null
|
|
371
|
+
/** 0 or 1: not every driver in this package can bind a boolean. */
|
|
372
|
+
ok: number | null
|
|
373
|
+
/** JSON-encoded, so a brace-leading response is not read back as an object. */
|
|
374
|
+
response: string | null
|
|
375
|
+
/** JSON array, null when the turn produced no finding. */
|
|
376
|
+
findingKinds: string | null
|
|
377
|
+
tokensIn: Generated<number>
|
|
378
|
+
tokensOut: Generated<number>
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* One persona's cadence. Nullable timestamps and 0/1 flags for the same driver
|
|
383
|
+
* reasons as {@link VirtualUserRunTable}.
|
|
384
|
+
*/
|
|
385
|
+
export interface VirtualUserScheduleTable {
|
|
386
|
+
persona: string
|
|
387
|
+
/** 0 or 1: not every driver in this package can bind a boolean. */
|
|
388
|
+
enabled: Generated<number>
|
|
389
|
+
disposition: string
|
|
390
|
+
goals: Generated<string>
|
|
391
|
+
/** JSON, or null for the engine's own default budget. */
|
|
392
|
+
budget: string | null
|
|
393
|
+
/** BIGINT, so a driver may hand either back. */
|
|
394
|
+
minIntervalMs: string | number | bigint
|
|
395
|
+
maxIntervalMs: string | number | bigint
|
|
396
|
+
nextRunAt: Date | string
|
|
397
|
+
lastRunId: string | null
|
|
398
|
+
lastRunAt: Date | string | null
|
|
399
|
+
}
|
|
400
|
+
|
|
355
401
|
export interface KyselyPikkuDB {
|
|
356
402
|
pikkuScopes: PikkuScopesTable
|
|
357
403
|
pikkuRoles: PikkuRolesTable
|
|
@@ -382,5 +428,7 @@ export interface KyselyPikkuDB {
|
|
|
382
428
|
webhookDelivery: WebhookDeliveryTable
|
|
383
429
|
webhookDeliveryAttempt: WebhookDeliveryAttemptTable
|
|
384
430
|
virtualUserRun: VirtualUserRunTable
|
|
431
|
+
virtualUserRunStep: VirtualUserRunStepTable
|
|
432
|
+
virtualUserSchedule: VirtualUserScheduleTable
|
|
385
433
|
audit: AuditTable
|
|
386
434
|
}
|
|
@@ -39,6 +39,28 @@ const FINDING = {
|
|
|
39
39
|
step: 7,
|
|
40
40
|
} as any
|
|
41
41
|
|
|
42
|
+
const INTENT = {
|
|
43
|
+
id: 'intent_1',
|
|
44
|
+
sourceId: 'exportADoc',
|
|
45
|
+
title: 'Export a doc',
|
|
46
|
+
status: 'done',
|
|
47
|
+
steps: [0, 1],
|
|
48
|
+
suspensions: 0,
|
|
49
|
+
} as any
|
|
50
|
+
|
|
51
|
+
const step = (index: number, over: Record<string, unknown> = {}) =>
|
|
52
|
+
({
|
|
53
|
+
index,
|
|
54
|
+
intentId: 'intent_1',
|
|
55
|
+
action: { kind: 'call', rpcName: 'listDocs', input: {} },
|
|
56
|
+
status: 200,
|
|
57
|
+
ok: true,
|
|
58
|
+
response: '{"docs":[]}',
|
|
59
|
+
tokensIn: 40,
|
|
60
|
+
tokensOut: 12,
|
|
61
|
+
...over,
|
|
62
|
+
}) as any
|
|
63
|
+
|
|
42
64
|
describe('KyselyVirtualUserRunStore', () => {
|
|
43
65
|
let db: Kysely<KyselyPikkuDB>
|
|
44
66
|
let store: KyselyVirtualUserRunStore
|
|
@@ -114,6 +136,8 @@ describe('KyselyVirtualUserRunStore', () => {
|
|
|
114
136
|
tally: TALLY,
|
|
115
137
|
memory: { docId: 'doc_1', orgId: 'org_9' },
|
|
116
138
|
stoppedBy: 'budget-steps',
|
|
139
|
+
intents: [INTENT],
|
|
140
|
+
steps: [step(0), step(1)],
|
|
117
141
|
})
|
|
118
142
|
|
|
119
143
|
const run = await store.get(runId)
|
|
@@ -182,14 +206,17 @@ describe('KyselyVirtualUserRunStore', () => {
|
|
|
182
206
|
assert.equal(runs[0]?.persona, 'susan')
|
|
183
207
|
})
|
|
184
208
|
|
|
185
|
-
test('completing an unknown run does not throw', async () => {
|
|
209
|
+
test('completing an unknown run does not throw, and leaves no steps behind', async () => {
|
|
186
210
|
await store.init()
|
|
187
211
|
await store.complete('nope', {
|
|
188
212
|
findings: [],
|
|
189
213
|
tally: TALLY,
|
|
190
214
|
memory: {},
|
|
191
215
|
stoppedBy: 'exhausted',
|
|
216
|
+
intents: [INTENT],
|
|
217
|
+
steps: [step(0)],
|
|
192
218
|
})
|
|
219
|
+
assert.deepEqual(await store.steps('nope'), [])
|
|
193
220
|
})
|
|
194
221
|
|
|
195
222
|
// Not every project installs SerializePlugin, and a bare SQLite driver cannot
|
|
@@ -214,6 +241,8 @@ describe('KyselyVirtualUserRunStore', () => {
|
|
|
214
241
|
tally: TALLY,
|
|
215
242
|
memory: { docId: 'doc_1' },
|
|
216
243
|
stoppedBy: 'exhausted',
|
|
244
|
+
intents: [INTENT],
|
|
245
|
+
steps: [step(0)],
|
|
217
246
|
})
|
|
218
247
|
|
|
219
248
|
const run = await bareStore.get(runId)
|
|
@@ -224,5 +253,117 @@ describe('KyselyVirtualUserRunStore', () => {
|
|
|
224
253
|
assert.equal(run?.findings[0]?.detail, FINDING.detail)
|
|
225
254
|
assert.ok(run?.createdAt instanceof Date)
|
|
226
255
|
assert.ok(run?.finishedAt instanceof Date)
|
|
256
|
+
assert.deepEqual(await bareStore.steps(runId), [step(0)])
|
|
257
|
+
})
|
|
258
|
+
|
|
259
|
+
test('the transcript comes back in the order it happened', async () => {
|
|
260
|
+
const runId = await store.start({
|
|
261
|
+
persona: 'susan',
|
|
262
|
+
disposition: 'realistic',
|
|
263
|
+
seed: 1,
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
await store.complete(runId, {
|
|
267
|
+
findings: [],
|
|
268
|
+
tally: TALLY,
|
|
269
|
+
memory: {},
|
|
270
|
+
stoppedBy: 'exhausted',
|
|
271
|
+
intents: [INTENT],
|
|
272
|
+
steps: [step(2), step(0), step(1)],
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
const steps = await store.steps(runId)
|
|
276
|
+
assert.deepEqual(
|
|
277
|
+
steps.map((one) => one.index),
|
|
278
|
+
[0, 1, 2]
|
|
279
|
+
)
|
|
280
|
+
assert.deepEqual(steps[0], step(0))
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
// The turn the model got wrong is the one worth reading, and it carries none
|
|
284
|
+
// of the fields a successful call does.
|
|
285
|
+
test('a turn with no response and no status survives the round trip', async () => {
|
|
286
|
+
const runId = await store.start({
|
|
287
|
+
persona: 'susan',
|
|
288
|
+
disposition: 'careless',
|
|
289
|
+
seed: 1,
|
|
290
|
+
})
|
|
291
|
+
|
|
292
|
+
const invalid = step(0, {
|
|
293
|
+
intentId: undefined,
|
|
294
|
+
action: { kind: 'invalid', detail: 'no such rpc: listDoc' },
|
|
295
|
+
status: undefined,
|
|
296
|
+
ok: undefined,
|
|
297
|
+
response: undefined,
|
|
298
|
+
findingKinds: ['invalid-action'],
|
|
299
|
+
})
|
|
300
|
+
|
|
301
|
+
await store.complete(runId, {
|
|
302
|
+
findings: [],
|
|
303
|
+
tally: TALLY,
|
|
304
|
+
memory: {},
|
|
305
|
+
stoppedBy: 'exhausted',
|
|
306
|
+
intents: [],
|
|
307
|
+
steps: [invalid],
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
const [stored] = await store.steps(runId)
|
|
311
|
+
assert.deepEqual(stored, {
|
|
312
|
+
index: 0,
|
|
313
|
+
action: { kind: 'invalid', detail: 'no such rpc: listDoc' },
|
|
314
|
+
findingKinds: ['invalid-action'],
|
|
315
|
+
tokensIn: 40,
|
|
316
|
+
tokensOut: 12,
|
|
317
|
+
})
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
// Ten columns times a 500-step budget is five thousand bound variables, and a
|
|
321
|
+
// bare sqlite driver takes 999 — so the long run is exactly the one that must
|
|
322
|
+
// not be lost.
|
|
323
|
+
test('a run longer than one insert can hold is stored whole', async () => {
|
|
324
|
+
const runId = await store.start({
|
|
325
|
+
persona: 'susan',
|
|
326
|
+
disposition: 'adversarial',
|
|
327
|
+
seed: 1,
|
|
328
|
+
})
|
|
329
|
+
|
|
330
|
+
const steps = Array.from({ length: 240 }, (_, index) => step(index))
|
|
331
|
+
await store.complete(runId, {
|
|
332
|
+
findings: [],
|
|
333
|
+
tally: TALLY,
|
|
334
|
+
memory: {},
|
|
335
|
+
stoppedBy: 'budget-steps',
|
|
336
|
+
intents: [],
|
|
337
|
+
steps,
|
|
338
|
+
})
|
|
339
|
+
|
|
340
|
+
assert.equal((await store.steps(runId, { limit: 500 })).length, 240)
|
|
341
|
+
assert.deepEqual(
|
|
342
|
+
(await store.steps(runId, { limit: 2, offset: 100 })).map(
|
|
343
|
+
(one) => one.index
|
|
344
|
+
),
|
|
345
|
+
[100, 101]
|
|
346
|
+
)
|
|
347
|
+
})
|
|
348
|
+
|
|
349
|
+
test('intents ride on the run, not on the transcript', async () => {
|
|
350
|
+
const runId = await store.start({
|
|
351
|
+
persona: 'susan',
|
|
352
|
+
disposition: 'realistic',
|
|
353
|
+
seed: 1,
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
await store.complete(runId, {
|
|
357
|
+
findings: [],
|
|
358
|
+
tally: TALLY,
|
|
359
|
+
memory: {},
|
|
360
|
+
stoppedBy: 'exhausted',
|
|
361
|
+
intents: [INTENT],
|
|
362
|
+
steps: [],
|
|
363
|
+
})
|
|
364
|
+
|
|
365
|
+
const [listed] = await store.list({ persona: 'susan' })
|
|
366
|
+
assert.deepEqual(listed?.intents, [INTENT])
|
|
367
|
+
assert.deepEqual(await store.steps(runId), [])
|
|
227
368
|
})
|
|
228
369
|
})
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
import type { Kysely, Selectable } from 'kysely'
|
|
2
2
|
import { randomUUID } from 'node:crypto'
|
|
3
|
-
import type {
|
|
3
|
+
import type {
|
|
4
|
+
StepRecord,
|
|
5
|
+
VirtualUserDisposition,
|
|
6
|
+
} from '@pikku/core/virtual-user'
|
|
4
7
|
import type {
|
|
5
8
|
VirtualUserRunOutcome,
|
|
6
9
|
VirtualUserRunRecord,
|
|
7
10
|
VirtualUserRunStart,
|
|
8
11
|
VirtualUserRunStore,
|
|
9
12
|
} from '@pikku/core/virtual-user'
|
|
10
|
-
import type {
|
|
13
|
+
import type {
|
|
14
|
+
KyselyPikkuDB,
|
|
15
|
+
VirtualUserRunStepTable,
|
|
16
|
+
VirtualUserRunTable,
|
|
17
|
+
} from './kysely-tables.js'
|
|
11
18
|
import { parseJson } from './kysely-json.js'
|
|
12
19
|
import { requirePikkuSchema } from './schema/index.js'
|
|
13
20
|
import { virtualUserSchema } from './schema/virtual-user.schema.js'
|
|
@@ -31,6 +38,16 @@ import { virtualUserSchema } from './schema/virtual-user.schema.js'
|
|
|
31
38
|
* ids. This is a privileged store; the scaffolded reads are scope-gated, and
|
|
32
39
|
* exposing these rows more widely publishes your own exploits.
|
|
33
40
|
*/
|
|
41
|
+
/**
|
|
42
|
+
* How many step rows go in one insert.
|
|
43
|
+
*
|
|
44
|
+
* Not a performance dial. A bare SQLite driver binds at most 999 variables per
|
|
45
|
+
* statement, and ten columns times a 500-step budget is five thousand — so an
|
|
46
|
+
* un-chunked insert does not fail on a long run, it fails on the *interesting*
|
|
47
|
+
* long run, which is the one nobody wants to lose.
|
|
48
|
+
*/
|
|
49
|
+
const STEP_INSERT_CHUNK = 50
|
|
50
|
+
|
|
34
51
|
export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
35
52
|
private initialized = false
|
|
36
53
|
|
|
@@ -71,7 +88,7 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
|
71
88
|
|
|
72
89
|
async complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void> {
|
|
73
90
|
await this.init()
|
|
74
|
-
await this.db
|
|
91
|
+
const updated = await this.db
|
|
75
92
|
.updateTable('virtualUserRun')
|
|
76
93
|
.set({
|
|
77
94
|
status: 'completed',
|
|
@@ -80,11 +97,16 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
|
80
97
|
// Overwritten rather than merged: the engine's memory already carries
|
|
81
98
|
// what it was given, plus what it learned on the way.
|
|
82
99
|
memory: JSON.stringify(outcome.memory),
|
|
100
|
+
intents: JSON.stringify(outcome.intents),
|
|
83
101
|
stoppedBy: outcome.stoppedBy,
|
|
84
102
|
finishedAt: new Date().toISOString(),
|
|
85
103
|
})
|
|
86
104
|
.where('runId', '=', runId)
|
|
87
|
-
.
|
|
105
|
+
.executeTakeFirst()
|
|
106
|
+
// Nothing owns steps written against a run that is not there, and nothing
|
|
107
|
+
// would ever read or reap them — there is no foreign key to refuse them.
|
|
108
|
+
if (updated.numUpdatedRows === 0n) return
|
|
109
|
+
await this.writeSteps(runId, outcome.steps)
|
|
88
110
|
}
|
|
89
111
|
|
|
90
112
|
async fail(runId: string, error: string): Promise<void> {
|
|
@@ -128,6 +150,69 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
|
128
150
|
return rows.map((row) => this.toRecord(row))
|
|
129
151
|
}
|
|
130
152
|
|
|
153
|
+
async steps(
|
|
154
|
+
runId: string,
|
|
155
|
+
options?: { limit?: number; offset?: number }
|
|
156
|
+
): Promise<StepRecord[]> {
|
|
157
|
+
await this.init()
|
|
158
|
+
const rows = await this.db
|
|
159
|
+
.selectFrom('virtualUserRunStep')
|
|
160
|
+
.selectAll()
|
|
161
|
+
.where('runId', '=', runId)
|
|
162
|
+
.orderBy('stepIndex', 'asc')
|
|
163
|
+
.limit(options?.limit ?? 1000)
|
|
164
|
+
.offset(options?.offset ?? 0)
|
|
165
|
+
.execute()
|
|
166
|
+
return rows.map((row) => this.toStep(row))
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
private async writeSteps(
|
|
170
|
+
runId: string,
|
|
171
|
+
steps: readonly StepRecord[]
|
|
172
|
+
): Promise<void> {
|
|
173
|
+
for (let at = 0; at < steps.length; at += STEP_INSERT_CHUNK) {
|
|
174
|
+
const chunk = steps.slice(at, at + STEP_INSERT_CHUNK)
|
|
175
|
+
await this.db
|
|
176
|
+
.insertInto('virtualUserRunStep')
|
|
177
|
+
.values(
|
|
178
|
+
chunk.map((step) => ({
|
|
179
|
+
runId,
|
|
180
|
+
stepIndex: step.index,
|
|
181
|
+
intentId: step.intentId ?? null,
|
|
182
|
+
action: JSON.stringify(step.action),
|
|
183
|
+
status: step.status ?? null,
|
|
184
|
+
ok: step.ok === undefined ? null : step.ok ? 1 : 0,
|
|
185
|
+
response:
|
|
186
|
+
step.response === undefined
|
|
187
|
+
? null
|
|
188
|
+
: JSON.stringify(step.response),
|
|
189
|
+
findingKinds: step.findingKinds
|
|
190
|
+
? JSON.stringify(step.findingKinds)
|
|
191
|
+
: null,
|
|
192
|
+
tokensIn: step.tokensIn,
|
|
193
|
+
tokensOut: step.tokensOut,
|
|
194
|
+
}))
|
|
195
|
+
)
|
|
196
|
+
.execute()
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private toStep(row: Selectable<VirtualUserRunStepTable>): StepRecord {
|
|
201
|
+
return {
|
|
202
|
+
index: Number(row.stepIndex),
|
|
203
|
+
...(row.intentId ? { intentId: row.intentId } : {}),
|
|
204
|
+
action: parseJson(row.action),
|
|
205
|
+
...(row.status === null ? {} : { status: Number(row.status) }),
|
|
206
|
+
...(row.ok === null ? {} : { ok: Boolean(row.ok) }),
|
|
207
|
+
...(row.response === null ? {} : { response: parseJson(row.response) }),
|
|
208
|
+
...(row.findingKinds
|
|
209
|
+
? { findingKinds: parseJson(row.findingKinds) }
|
|
210
|
+
: {}),
|
|
211
|
+
tokensIn: Number(row.tokensIn),
|
|
212
|
+
tokensOut: Number(row.tokensOut),
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
131
216
|
private toRecord(row: Selectable<VirtualUserRunTable>): VirtualUserRunRecord {
|
|
132
217
|
return {
|
|
133
218
|
runId: row.runId,
|
|
@@ -138,6 +223,7 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
|
|
|
138
223
|
goals: parseJson(row.goals) ?? [],
|
|
139
224
|
memory: parseJson(row.memory) ?? {},
|
|
140
225
|
findings: parseJson(row.findings) ?? [],
|
|
226
|
+
intents: parseJson(row.intents) ?? [],
|
|
141
227
|
tally: row.tally ? (parseJson(row.tally) ?? null) : null,
|
|
142
228
|
stoppedBy: row.stoppedBy,
|
|
143
229
|
error: row.error,
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { describe, test, beforeEach } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import Database from 'better-sqlite3'
|
|
4
|
+
import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
|
|
5
|
+
|
|
6
|
+
import { KyselyVirtualUserScheduleStore } from './kysely-virtual-user-schedule-store.js'
|
|
7
|
+
import { SerializePlugin } from './serialize-plugin.js'
|
|
8
|
+
import { applyPikkuSchemas, virtualUserScheduleSchema } from './schema/index.js'
|
|
9
|
+
import type { KyselyPikkuDB } from './kysely-tables.js'
|
|
10
|
+
|
|
11
|
+
// With the plugin, because most projects in this package run with it and it is
|
|
12
|
+
// the harder case: it deserialises columns the store has already stringified.
|
|
13
|
+
const withPlugin = () =>
|
|
14
|
+
new Kysely<KyselyPikkuDB>({
|
|
15
|
+
dialect: new SqliteDialect({ database: new Database(':memory:') }),
|
|
16
|
+
plugins: [new CamelCasePlugin(), new SerializePlugin()],
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
const bare = () =>
|
|
20
|
+
new Kysely<KyselyPikkuDB>({
|
|
21
|
+
dialect: new SqliteDialect({ database: new Database(':memory:') }),
|
|
22
|
+
plugins: [new CamelCasePlugin()],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
const HOUR = 60 * 60 * 1000
|
|
26
|
+
|
|
27
|
+
describe('KyselyVirtualUserScheduleStore', () => {
|
|
28
|
+
let db: Kysely<KyselyPikkuDB>
|
|
29
|
+
let store: KyselyVirtualUserScheduleStore
|
|
30
|
+
|
|
31
|
+
beforeEach(async () => {
|
|
32
|
+
db = withPlugin()
|
|
33
|
+
await applyPikkuSchemas(db, [virtualUserScheduleSchema])
|
|
34
|
+
store = new KyselyVirtualUserScheduleStore(db)
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
test('a persona given a cadence is not running until someone says so', async () => {
|
|
38
|
+
const written = await store.set({ persona: 'guest' })
|
|
39
|
+
assert.equal(written.enabled, false)
|
|
40
|
+
assert.deepEqual(await store.due(new Date(Date.now() + 1000)), [])
|
|
41
|
+
assert.equal((await store.get('guest'))?.enabled, false)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
test('a schedule written without an interval gets one, sparse by default', async () => {
|
|
45
|
+
const written = await store.set({ persona: 'guest' })
|
|
46
|
+
assert.ok(written.minIntervalMs >= HOUR)
|
|
47
|
+
assert.ok(written.maxIntervalMs >= written.minIntervalMs)
|
|
48
|
+
const stored = await store.get('guest')
|
|
49
|
+
assert.equal(stored?.minIntervalMs, written.minIntervalMs)
|
|
50
|
+
assert.equal(stored?.maxIntervalMs, written.maxIntervalMs)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
test('setting one field leaves the rest of the schedule alone', async () => {
|
|
54
|
+
await store.set({
|
|
55
|
+
persona: 'guest',
|
|
56
|
+
enabled: true,
|
|
57
|
+
disposition: 'adversarial',
|
|
58
|
+
goals: ['book a room'],
|
|
59
|
+
budget: { steps: 40 },
|
|
60
|
+
minIntervalMs: HOUR,
|
|
61
|
+
maxIntervalMs: 2 * HOUR,
|
|
62
|
+
})
|
|
63
|
+
await store.set({ persona: 'guest', enabled: false })
|
|
64
|
+
|
|
65
|
+
const stored = await store.get('guest')
|
|
66
|
+
assert.equal(stored?.enabled, false)
|
|
67
|
+
assert.equal(stored?.disposition, 'adversarial')
|
|
68
|
+
assert.deepEqual(stored?.goals, ['book a room'])
|
|
69
|
+
assert.deepEqual(stored?.budget, { steps: 40 })
|
|
70
|
+
assert.equal(stored?.minIntervalMs, HOUR)
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
test('only enabled personas whose time has come come back as due', async () => {
|
|
74
|
+
const now = new Date()
|
|
75
|
+
await store.set({
|
|
76
|
+
persona: 'due',
|
|
77
|
+
enabled: true,
|
|
78
|
+
nextRunAt: new Date(now.getTime() - HOUR),
|
|
79
|
+
})
|
|
80
|
+
await store.set({
|
|
81
|
+
persona: 'later',
|
|
82
|
+
enabled: true,
|
|
83
|
+
nextRunAt: new Date(now.getTime() + HOUR),
|
|
84
|
+
})
|
|
85
|
+
await store.set({
|
|
86
|
+
persona: 'off',
|
|
87
|
+
enabled: false,
|
|
88
|
+
nextRunAt: new Date(now.getTime() - HOUR),
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
assert.deepEqual(
|
|
92
|
+
(await store.due(now)).map((row) => row.persona),
|
|
93
|
+
['due']
|
|
94
|
+
)
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
test('claiming pushes the persona out of the due list', async () => {
|
|
98
|
+
const now = new Date()
|
|
99
|
+
const due = new Date(now.getTime() - HOUR)
|
|
100
|
+
await store.set({ persona: 'guest', enabled: true, nextRunAt: due })
|
|
101
|
+
const acquired = await store.claim('guest', {
|
|
102
|
+
from: due,
|
|
103
|
+
nextRunAt: new Date(now.getTime() + HOUR),
|
|
104
|
+
runId: 'run-1',
|
|
105
|
+
at: now,
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
assert.equal(acquired, true)
|
|
109
|
+
|
|
110
|
+
assert.deepEqual(await store.due(now), [])
|
|
111
|
+
const stored = await store.get('guest')
|
|
112
|
+
assert.equal(stored?.lastRunId, 'run-1')
|
|
113
|
+
assert.equal(stored?.lastRunAt?.getTime(), now.getTime())
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
test('the claim written before a run starts does not blank the last run', async () => {
|
|
117
|
+
const now = new Date()
|
|
118
|
+
const written = await store.set({ persona: 'guest', enabled: true })
|
|
119
|
+
await store.claim('guest', {
|
|
120
|
+
from: written.nextRunAt,
|
|
121
|
+
nextRunAt: now,
|
|
122
|
+
runId: 'run-1',
|
|
123
|
+
at: now,
|
|
124
|
+
})
|
|
125
|
+
await store.claim('guest', {
|
|
126
|
+
from: now,
|
|
127
|
+
nextRunAt: new Date(now.getTime() + HOUR),
|
|
128
|
+
runId: null,
|
|
129
|
+
at: now,
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
assert.equal((await store.get('guest'))?.lastRunId, 'run-1')
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
test('only one of two ticks holding the same due row can claim it', async () => {
|
|
136
|
+
const now = new Date()
|
|
137
|
+
const due = new Date(now.getTime() - HOUR)
|
|
138
|
+
await store.set({ persona: 'guest', enabled: true, nextRunAt: due })
|
|
139
|
+
|
|
140
|
+
const [first] = await store.due(now)
|
|
141
|
+
const [second] = await store.due(now)
|
|
142
|
+
assert.equal(first?.nextRunAt.getTime(), second?.nextRunAt.getTime())
|
|
143
|
+
|
|
144
|
+
const won = await store.claim('guest', {
|
|
145
|
+
from: first!.nextRunAt,
|
|
146
|
+
nextRunAt: new Date(now.getTime() + HOUR),
|
|
147
|
+
runId: 'run-1',
|
|
148
|
+
at: now,
|
|
149
|
+
})
|
|
150
|
+
const lost = await store.claim('guest', {
|
|
151
|
+
from: second!.nextRunAt,
|
|
152
|
+
nextRunAt: new Date(now.getTime() + 2 * HOUR),
|
|
153
|
+
runId: 'run-2',
|
|
154
|
+
at: now,
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
assert.deepEqual([won, lost], [true, false])
|
|
158
|
+
assert.equal((await store.get('guest'))?.lastRunId, 'run-1')
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
test('a cadence survives a store with no serialize plugin', async () => {
|
|
162
|
+
const plainDb = bare()
|
|
163
|
+
await applyPikkuSchemas(plainDb, [virtualUserScheduleSchema])
|
|
164
|
+
const plain = new KyselyVirtualUserScheduleStore(plainDb)
|
|
165
|
+
await plain.set({
|
|
166
|
+
persona: 'guest',
|
|
167
|
+
enabled: true,
|
|
168
|
+
goals: ['book a room'],
|
|
169
|
+
budget: { steps: 40, duration: '30m' },
|
|
170
|
+
minIntervalMs: HOUR,
|
|
171
|
+
maxIntervalMs: 2 * HOUR,
|
|
172
|
+
})
|
|
173
|
+
|
|
174
|
+
const stored = await plain.get('guest')
|
|
175
|
+
assert.equal(stored?.enabled, true)
|
|
176
|
+
assert.deepEqual(stored?.goals, ['book a room'])
|
|
177
|
+
assert.deepEqual(stored?.budget, { steps: 40, duration: '30m' })
|
|
178
|
+
assert.ok(stored?.nextRunAt instanceof Date)
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
test('a persona can be taken off the clock entirely', async () => {
|
|
182
|
+
await store.set({ persona: 'guest', enabled: true })
|
|
183
|
+
await store.remove('guest')
|
|
184
|
+
assert.equal(await store.get('guest'), null)
|
|
185
|
+
assert.deepEqual(await store.list(), [])
|
|
186
|
+
})
|
|
187
|
+
})
|