@pikku/kysely 0.13.19 → 0.13.21

package/src/schema/schema.test.ts

@@ -14,9 +14,10 @@ import {
 import Database from 'better-sqlite3'
 import {
   pikkuSchemas,
+  requiredPikkuSchemas,
   compilePikkuSchemas,
   applyPikkuSchemas,
-  ensurePikkuSchema,
+  requirePikkuSchema,
   declaredTables,
   unsatisfiedRequirements,
   type PikkuSchema,
@@ -238,17 +239,32 @@ describe('pikku runtime schema — prerequisites', () => {
   })
 })
 
-describe('ensurePikkuSchema — what a service does at boot', () => {
+describe('requirePikkuSchema — what a service does at boot', () => {
   const memoryDb = () =>
     new Kysely<any>({
       dialect: new SqliteDialect({ database: new Database(':memory:') }),
     })
 
-  test('creates the tables the first time and finds them the second', async () => {
+  test('refuses a database the migration never reached, naming the command', async () => {
     const db = memoryDb()
     try {
-      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'created')
-      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+      await assert.rejects(
+        requirePikkuSchema(db, workflowSchema),
+        /The 'workflow' schema is not in this database.*pikku db generate/s
+      )
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('creates nothing on the way to that refusal', async () => {
+    // The whole point: a service that finds its tables missing stops, and the
+    // database it stopped against is exactly as it was.
+    const db = memoryDb()
+    try {
+      await assert.rejects(requirePikkuSchema(db, workflowSchema))
+      const tables = await db.introspection.getTables()
+      assert.deepEqual(tables, [])
     } finally {
       await db.destroy()
     }
@@ -256,21 +272,22 @@ describe('ensurePikkuSchema — what a service does at boot', () => {
 
   test('two services declaring the same schema do not fight over it', async () => {
     // The workflow service and the workflow mirror both own these tables, and
-    // both call init(). Whoever runs second must find them and stop, rather
-    // than issuing DDL the database will reject.
+    // both call init(). Neither creates anything, so both find the migrated
+    // tables and return.
     const db = memoryDb()
     try {
-      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'created')
-      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+      await applyPikkuSchemas(db, [workflowSchema])
+      await assert.doesNotReject(requirePikkuSchema(db, workflowSchema))
+      await assert.doesNotReject(requirePikkuSchema(db, workflowSchema))
     } finally {
       await db.destroy()
     }
   })
 
-  test('refuses a half-applied schema rather than filling in the rest', async () => {
+  test('a half-applied schema is refused the same way a missing one is', async () => {
     // Something else already owns part of it — a migration, an older release,
-    // a hand-run script. Creating the remainder at boot leaves two authorities
-    // over one set of tables, which is the condition all of this exists to end.
+    // a hand-run script. Half-present is not a distinct case: the remedy is the
+    // same migration either way, so the sentence names both halves and stops.
     const db = memoryDb()
     try {
       await db.schema
@@ -279,8 +296,8 @@ describe('ensurePikkuSchema — what a service does at boot', () => {
         .execute()
 
       await assert.rejects(
-        ensurePikkuSchema(db, workflowSchema),
-        /The 'workflow' schema is half applied/
+        requirePikkuSchema(db, workflowSchema),
+        /workflow_step.*missing.*workflow_runs already there/s
       )
     } finally {
       await db.destroy()
@@ -288,7 +305,7 @@ describe('ensurePikkuSchema — what a service does at boot', () => {
   })
 })
 
-describe('ensurePikkuSchema — a connection bound to a schema', () => {
+describe('requirePikkuSchema — a connection bound to a schema', () => {
   const WORKFLOW_TABLES = [
     'workflow_runs',
     'workflow_step',
@@ -301,7 +318,7 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
    *
    * Stubbed rather than run against a real database because sqlite has one
    * schema and nothing else here speaks postgres — and introspection is the
-   * only thing `ensurePikkuSchema` reads before it decides. DummyDriver takes
+   * only thing `requirePikkuSchema` reads before it decides. DummyDriver takes
    * whatever DDL follows.
    */
   const introspecting = (
@@ -352,9 +369,8 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
       WORKFLOW_TABLES.map((name) => ({ schema: 'app', name }))
     )
 
-    assert.equal(
-      await ensurePikkuSchema(db.withSchema('app'), workflowSchema),
-      'present'
+    await assert.doesNotReject(
+      requirePikkuSchema(db.withSchema('app'), workflowSchema)
     )
   })
 
@@ -365,9 +381,9 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
       WORKFLOW_TABLES.map((name) => ({ schema: 'public', name }))
     )
 
-    assert.equal(
-      await ensurePikkuSchema(db.withSchema('app'), workflowSchema),
-      'created'
+    await assert.rejects(
+      requirePikkuSchema(db.withSchema('app'), workflowSchema),
+      /is not in this database/
     )
   })
 
@@ -376,7 +392,7 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
       WORKFLOW_TABLES.map((name) => ({ schema: 'public', name }))
     )
 
-    assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+    await assert.doesNotReject(requirePikkuSchema(db, workflowSchema))
   })
 
   test('matches on the bare name when the engine reports no schemas', async () => {
@@ -387,9 +403,8 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
       WORKFLOW_TABLES.map((name) => ({ schema: undefined, name }))
     )
 
-    assert.equal(
-      await ensurePikkuSchema(db.withSchema('main'), workflowSchema),
-      'present'
+    await assert.doesNotReject(
+      requirePikkuSchema(db.withSchema('main'), workflowSchema)
     )
   })
 
@@ -414,7 +429,7 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
     })
 
     await assert.rejects(
-      ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      applyPikkuSchemas(db.withSchema('app'), [workflowSchema]),
       (error: Error) => {
         assert.equal(error, pgError, 'the engine error must pass through as-is')
         return true
@@ -431,7 +446,7 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
     })
     try {
       await assert.rejects(
-        ensurePikkuSchema(db.withSchema('main'), workflowSchema),
+        applyPikkuSchemas(db.withSchema('main'), [workflowSchema]),
         (error: Error) => {
           assert.match(error.message, /bound to a schema/)
           assert.match(error.message, /takes a bare table name/i)
@@ -448,7 +463,7 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
     const db = introspecting([{ schema: 'app', name: 'workflow_runs' }])
 
     await assert.rejects(
-      ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      requirePikkuSchema(db.withSchema('app'), workflowSchema),
       /app\.workflow_step.*missing.*app\.workflow_runs already there/s
     )
   })
@@ -532,3 +547,45 @@ describe('compiling into a schema', () => {
     )
   })
 })
+
+describe('requiredPikkuSchemas', () => {
+  test('a project that reaches no service still gets the ungated schemas', () => {
+    const names = requiredPikkuSchemas(new Set()).map((s) => s.name)
+    assert.deepEqual(names, ['session', 'secret', 'deployment'])
+  })
+
+  test('a service brings in its own schema, in the declared order', () => {
+    const names = requiredPikkuSchemas(
+      new Set(['channelStore', 'workflowService'])
+    ).map((s) => s.name)
+    assert.deepEqual(names, [
+      'channel',
+      'session',
+      'secret',
+      'deployment',
+      'workflow',
+    ])
+  })
+
+  test('any one of a schema s owners is enough to ask for it', () => {
+    // A schema several services share is needed as soon as one of them is.
+    assert.ok(
+      requiredPikkuSchemas(new Set(['workflowRunService'])).some(
+        (s) => s.name === 'workflow'
+      )
+    )
+    assert.ok(
+      requiredPikkuSchemas(new Set(['eventHub'])).some(
+        (s) => s.name === 'channel'
+      )
+    )
+  })
+
+  test('naming every owner asks for everything', () => {
+    const owners = pikkuSchemas.flatMap((s) => s.ownedBy ?? [])
+    assert.equal(
+      requiredPikkuSchemas(new Set(owners)).length,
+      pikkuSchemas.length
+    )
+  })
+})

package/src/schema/scope.schema.ts

@@ -19,6 +19,7 @@ import { requiredType, type PikkuSchema } from './pikku-schema.types.js'
  */
 export const scopeSchema: PikkuSchema = {
   name: 'scope',
+  ownedBy: ['scopeService'],
   requires: [{ table: 'user', column: 'id', owner: 'Better Auth' }],
   statements: [
     (db) =>

package/src/schema/virtual-user-schedule.schema.ts

@@ -0,0 +1,48 @@
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * The `virtualUserSchedule` table {@link KyselyVirtualUserScheduleStore} writes
+ * to: one row per persona that keeps using the app on its own.
+ *
+ * Its own schema rather than a third table in {@link virtualUserSchema}, and
+ * owned by its own store: recording what a persona did and deciding that it
+ * keeps going on its own are separate opt-ins, and a project that only wants
+ * the first should not be handed a cadence table it never writes to.
+ *
+ * `persona` is the primary key: a persona has one cadence, and a second row for
+ * the same one would mean two copies of the same user acting at once — not a
+ * heavier test, a different and unreproducible one.
+ */
+export const virtualUserScheduleSchema: PikkuSchema = {
+  name: 'virtual-user-schedule',
+  ownedBy: ['virtualUserScheduleStore'],
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('virtualUserSchedule')
+        .addColumn('persona', 'varchar(255)', (col) => col.primaryKey())
+        // 0 or 1 rather than a boolean, matching the run tables: a bare sqlite
+        // driver cannot bind one, and `SerializePlugin` is not installed
+        // everywhere. Off by default, so creating the table spends nothing.
+        .addColumn('enabled', 'integer', (col) => col.defaultTo(0).notNull())
+        .addColumn('disposition', 'varchar(50)', (col) => col.notNull())
+        .addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
+        .addColumn('budget', 'text')
+        // The gap to the next run is drawn between these rather than fixed, so
+        // the persona does not keep an appointment.
+        .addColumn('minIntervalMs', 'bigint', (col) => col.notNull())
+        .addColumn('maxIntervalMs', 'bigint', (col) => col.notNull())
+        // The entire schedule. Written before a run is dispatched, so a tick
+        // that dies halfway cannot leave the row due for the next one.
+        .addColumn('nextRunAt', 'timestamp', (col) => col.notNull())
+        .addColumn('lastRunId', 'varchar(36)')
+        .addColumn('lastRunAt', 'timestamp'),
+
+    // The only read that matters at speed: which rows are due, on every tick.
+    (db) =>
+      db.schema
+        .createIndex('idx_virtual_user_schedule_due')
+        .on('virtualUserSchedule')
+        .columns(['enabled', 'nextRunAt']),
+  ],
+}

package/src/schema/virtual-user.schema.ts

@@ -2,7 +2,8 @@ import { sql } from 'kysely'
 import type { PikkuSchema } from './pikku-schema.types.js'
 
 /**
- * The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
+ * The `virtualUserRun` and `virtualUserRunStep` tables
+ * {@link KyselyVirtualUserRunStore} writes to.
  *
  * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
  * a project that runs virtual users does, and creating it for everyone would
@@ -19,7 +20,8 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  * week's does not depend on which database they were found in.
  */
 export const virtualUserSchema: PikkuSchema = {
-  name: 'virtualUser',
+  name: 'virtual-user',
+  ownedBy: ['virtualUserRunStore'],
   statements: [
     (db) =>
       db.schema
@@ -37,6 +39,10 @@ export const virtualUserSchema: PikkuSchema = {
         .addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
         .addColumn('memory', 'text', (col) => col.defaultTo('{}').notNull())
         .addColumn('findings', 'text', (col) => col.defaultTo('[]').notNull())
+        // What the user set out to do and how far each one got. On the run row
+        // rather than beside the steps: there are a handful of them, and every
+        // read of the run wants them.
+        .addColumn('intents', 'text', (col) => col.defaultTo('[]').notNull())
         // Counts the engine kept: steps, calls, mutations, tokens, elapsed.
         .addColumn('tally', 'text')
         .addColumn('stoppedBy', 'varchar(255)')
@@ -61,5 +67,40 @@ export const virtualUserSchema: PikkuSchema = {
         .createIndex('idx_virtual_user_run_persona')
         .on('virtualUserRun')
         .columns(['persona', 'createdAt']),
+
+    // One row per turn, kept out of `virtualUserRun` so that listing runs does
+    // not drag a budget's worth of transcript along with it. No foreign key
+    // onto the run for the same reason `persona` is not one: this table is
+    // created by the store at boot on whatever engine the project uses, and a
+    // cascade that half the engines enforce differently is worse than the read
+    // rule that a step with no run is orphaned.
+    (db) =>
+      db.schema
+        .createTable('virtualUserRunStep')
+        .addColumn('runId', 'varchar(36)', (col) => col.notNull())
+        // The engine's own step number, which is what the intent records point
+        // at — not a surrogate key, so a transcript reads the same after paging.
+        // Spelled `stepIndex` because `index` is a reserved word in mysql.
+        .addColumn('stepIndex', 'integer', (col) => col.notNull())
+        .addColumn('intentId', 'varchar(255)')
+        // The action as the engine scheduled it, including the `invalid` shape
+        // for a turn the model got wrong — that turn is the interesting one.
+        .addColumn('action', 'text', (col) => col.notNull())
+        .addColumn('status', 'integer')
+        // 0 or 1 rather than a boolean, because a bare sqlite driver cannot
+        // bind one at all and `SerializePlugin` is not installed everywhere.
+        .addColumn('ok', 'integer')
+        // Already truncated by the engine, and stored JSON-encoded rather than
+        // raw: a truncated API response usually starts with a brace, which
+        // `SerializePlugin` would read back as an object rather than the string
+        // the engine actually saw.
+        .addColumn('response', 'text')
+        .addColumn('findingKinds', 'text')
+        .addColumn('tokensIn', 'integer', (col) => col.defaultTo(0).notNull())
+        .addColumn('tokensOut', 'integer', (col) => col.defaultTo(0).notNull())
+        .addPrimaryKeyConstraint('pk_virtual_user_run_step', [
+          'runId',
+          'stepIndex',
+        ]),
   ],
 }

package/src/schema/webhook.schema.ts

@@ -4,6 +4,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
 /** Webhook deliveries and one row per delivery attempt. */
 export const webhookSchema: PikkuSchema = {
   name: 'webhook',
+  ownedBy: ['webhookService'],
   statements: [
     (db) =>
       db.schema

package/src/schema/workflow.schema.ts

@@ -18,6 +18,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  */
 export const workflowSchema: PikkuSchema = {
   name: 'workflow',
+  ownedBy: ['workflowService', 'workflowRunService'],
   statements: [
     (db) =>
       db.schema