@pikku/kysely 0.13.7 → 0.13.11

package/src/kysely-secret-service.ts

@@ -1,4 +1,5 @@
-import type { SecretService } from '@pikku/core/services'
+import { createSecretValue, isSecretValue, type SecretValue } from '@pikku/core'
+import type { SecretService, SecretValues } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import {
@@ -112,7 +113,7 @@ export class KyselySecretService implements SecretService {
     return kek
   }
 
-  async getSecret<T = string>(key: string): Promise<T> {
+  async getSecret<T = string>(key: string): Promise<SecretValue<T>> {
     const row = await this.db
       .selectFrom('secrets')
       .select(['ciphertext', 'wrappedDek', 'keyVersion'])
@@ -124,7 +125,7 @@ export class KyselySecretService implements SecretService {
     const kek = await this.getKEK(row.keyVersion)
     const result = await envelopeDecrypt<T>(kek, row.ciphertext, row.wrappedDek)
     await this.logAudit(key, 'read')
-    return result
+    return createSecretValue(result)
   }
 
   async hasSecret(key: string): Promise<boolean> {
@@ -137,9 +138,11 @@ export class KyselySecretService implements SecretService {
   }
 
   async setSecret(key: string, value: unknown): Promise<void> {
+    // Encrypting the wrapper would store its redaction, not the secret —
+    // writing a secret back to the vault is exactly what this method is for.
     const { ciphertext, wrappedDEK } = await envelopeEncrypt(
       await this.getKEK(this.keyVersion),
-      value
+      isSecretValue(value) ? value.reveal() : value
     )
     const now = new Date().toISOString()
 
@@ -173,7 +176,7 @@ export class KyselySecretService implements SecretService {
 
   async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
     keys: (keyof T & string)[]
-  ): Promise<T> {
+  ): Promise<Partial<SecretValues<T>>> {
     const rows = await this.db
       .selectFrom('secrets')
       .select(['key', 'ciphertext', 'wrappedDek', 'keyVersion'])
@@ -184,10 +187,8 @@ export class KyselySecretService implements SecretService {
     for (const row of rows) {
       try {
         const kek = await this.getKEK(row.keyVersion)
-        out[row.key] = await envelopeDecrypt(
-          kek,
-          row.ciphertext,
-          row.wrappedDek
+        out[row.key] = createSecretValue(
+          await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek)
         )
       } catch (cause) {
         // A stored secret that fails to decrypt is never expected — it means the
@@ -203,7 +204,7 @@ export class KyselySecretService implements SecretService {
         )
       }
     }
-    return out as T
+    return out as Partial<SecretValues<T>>
   }
 
   async rotateKEK(): Promise<number> {

package/src/kysely-services.test.ts

@@ -199,7 +199,7 @@ function registerTests(
 
       const second = new KyselySecretService(getDb(), { key: kek })
       await second.init()
-      assert.equal(await second.getSecret('shared-salt'), 'value')
+      assert.equal((await second.getSecret('shared-salt')).reveal(), 'value')
 
       const rows = await getDb()
         .selectFrom('secretKekSalts')
@@ -227,7 +227,7 @@ function registerTests(
       const elapsed = performance.now() - start
 
       assert.equal(Object.keys(out).length, 50)
-      assert.equal(out['bulk-7'], 'value-7')
+      assert.equal(out['bulk-7']!.reveal(), 'value-7')
       assert.ok(
         elapsed < 400,
         `getSecrets over 50 secrets took ${elapsed.toFixed(0)}ms; one derivation per secret would cost seconds`
@@ -258,7 +258,7 @@ function registerTests(
       const elapsed = performance.now() - start
 
       assert.equal(rotated, 50)
-      assert.equal(await rotator.getSecret('rotate-7'), 'value-7')
+      assert.equal((await rotator.getSecret('rotate-7')).reveal(), 'value-7')
       assert.ok(
         elapsed < 700,
         `rotating 50 secrets took ${elapsed.toFixed(0)}ms; two derivations per secret would cost seconds`

package/src/kysely-tables.test.ts

@@ -5,7 +5,15 @@ import { join } from 'node:path'
 import { Kysely, SqliteDialect } from 'kysely'
 import Database from 'better-sqlite3'
 
-import { applyPikkuSchemas } from './schema/index.js'
+import { applyPikkuSchemas, ensurePikkuSchema } from './schema/index.js'
+import { virtualUserSchema } from './schema/virtual-user.schema.js'
+
+/**
+ * Opt-in schemas that still have an interface in `KyselyPikkuDB`, so the drift
+ * check covers them too. A table being opt-in says when it is created, not
+ * whether its type is allowed to disagree with its DDL.
+ */
+const OPTIONAL_SCHEMAS = [virtualUserSchema]
 
 /**
  * `KyselyPikkuDB` and the schema declaration describe the same tables, and
@@ -65,6 +73,9 @@ const declaredSchema = async (): Promise<Map<string, Set<string>>> => {
       .addColumn('id', 'text', (col) => col.primaryKey())
       .execute()
     await applyPikkuSchemas(db)
+    for (const schema of OPTIONAL_SCHEMAS) {
+      await ensurePikkuSchema(db, schema)
+    }
 
     const tables = await db.introspection.getTables()
     return new Map(

package/src/kysely-tables.ts

@@ -281,6 +281,37 @@ export interface WebhookDeliveryAttemptTable {
   createdAt: Generated<Date>
 }
 
+/**
+ * One virtual-user run. The JSON columns (`goals`, `memory`, `findings`,
+ * `tally`) are text the store serialises itself, so the row is byte-identical
+ * on every engine.
+ */
+export interface VirtualUserRunTable {
+  runId: string
+  persona: string
+  disposition: string
+  /**
+   * Read back as a string by some drivers — a BIGINT does not always fit a JS
+   * number — so the store narrows it rather than trusting the column type.
+   */
+  seed: string | number | bigint
+  status: Generated<'running' | 'completed' | 'failed'>
+  goals: Generated<string>
+  memory: Generated<string>
+  findings: Generated<string>
+  tally: string | null
+  stoppedBy: string | null
+  error: string | null
+  startedBy: string | null
+  /**
+   * Written as an ISO string and read back as whatever the driver returns — a
+   * bare SQLite driver cannot bind a `Date` at all, and the store normalises
+   * both ends.
+   */
+  createdAt: Generated<Date | string>
+  finishedAt: Date | string | null
+}
+
 export interface KyselyPikkuDB {
   pikkuScopes: PikkuScopesTable
   pikkuRoles: PikkuRolesTable
@@ -309,4 +340,5 @@ export interface KyselyPikkuDB {
   pikkuUserSessions: UserSessionsTable
   webhookDelivery: WebhookDeliveryTable
   webhookDeliveryAttempt: WebhookDeliveryAttemptTable
+  virtualUserRun: VirtualUserRunTable
 }

package/src/kysely-virtual-user-run-store.test.ts

@@ -0,0 +1,212 @@
+import { describe, test, beforeEach } from 'node:test'
+import assert from 'node:assert/strict'
+import Database from 'better-sqlite3'
+import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
+
+import { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
+import { SerializePlugin } from './serialize-plugin.js'
+import type { KyselyPikkuDB } from './kysely-tables.js'
+
+// SerializePlugin is here because most projects in this package run with it, and
+// it is the harder case: it deserialises JSON columns the store has already
+// stringified. The store has to survive that, and its absence.
+const newDb = () =>
+  new Kysely<KyselyPikkuDB>({
+    dialect: new SqliteDialect({ database: new Database(':memory:') }),
+    plugins: [new CamelCasePlugin(), new SerializePlugin()],
+  })
+
+const TALLY = {
+  steps: 12,
+  calls: 30,
+  mutations: 4,
+  tokensIn: 900,
+  tokensOut: 120,
+  elapsed: 4200,
+} as any
+
+const FINDING = {
+  kind: 'schema-mismatch',
+  detail: 'listDocs returned a doc with no title',
+  rpcName: 'listDocs',
+  step: 7,
+} as any
+
+describe('KyselyVirtualUserRunStore', () => {
+  let db: Kysely<KyselyPikkuDB>
+  let store: KyselyVirtualUserRunStore
+
+  beforeEach(() => {
+    db = newDb()
+    store = new KyselyVirtualUserRunStore(db)
+  })
+
+  // The runtime never needs this table, so it arrives with the feature that
+  // fills it rather than in every database — which means the first write has to
+  // create it.
+  test('creates its table on first use', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 42,
+    })
+    assert.ok(runId)
+  })
+
+  test('a run starts as running, with no findings and no finish time', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'careless',
+      seed: 7,
+      goals: ['find the export button'],
+      memory: { docId: 'doc_1' },
+      startedBy: 'user_1',
+    })
+
+    const run = await store.get(runId)
+    assert.equal(run?.status, 'running')
+    assert.deepEqual(run?.findings, [])
+    assert.equal(run?.finishedAt, null)
+    assert.equal(run?.error, null)
+    assert.deepEqual(run?.goals, ['find the export button'])
+    assert.deepEqual(run?.memory, { docId: 'doc_1' })
+    assert.equal(run?.startedBy, 'user_1')
+  })
+
+  // The seed is the whole reason a finding is reproducible, and a BIGINT comes
+  // back as a string on some drivers — so it has to survive the round trip as a
+  // number.
+  test('the seed round-trips as a number', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 2_147_483_646,
+    })
+    const run = await store.get(runId)
+    assert.strictEqual(run?.seed, 2_147_483_646)
+  })
+
+  test('completing a run stores what it found', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'auditor',
+      seed: 1,
+      memory: { docId: 'doc_1' },
+    })
+
+    await store.complete(runId, {
+      findings: [FINDING],
+      tally: TALLY,
+      memory: { docId: 'doc_1', orgId: 'org_9' },
+      stoppedBy: 'budget-steps',
+    })
+
+    const run = await store.get(runId)
+    assert.equal(run?.status, 'completed')
+    assert.equal(run?.findings.length, 1)
+    assert.equal(run?.findings[0]?.detail, FINDING.detail)
+    assert.deepEqual(run?.tally, TALLY)
+    assert.equal(run?.stoppedBy, 'budget-steps')
+    // Overwritten, not merged: the engine's memory already carries what it was
+    // given plus what it learned.
+    assert.deepEqual(run?.memory, { docId: 'doc_1', orgId: 'org_9' })
+    assert.ok(run?.finishedAt instanceof Date)
+  })
+
+  // A crashed run and a run that found nothing are different answers, and this
+  // record is the only place that distinction survives.
+  test('a failed run is failed, not an empty success', async () => {
+    const runId = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 1,
+    })
+
+    await store.fail(runId, 'SCENARIO_ACTOR_SECRET is not set')
+
+    const run = await store.get(runId)
+    assert.equal(run?.status, 'failed')
+    assert.equal(run?.error, 'SCENARIO_ACTOR_SECRET is not set')
+    assert.deepEqual(run?.findings, [])
+    assert.ok(run?.finishedAt instanceof Date)
+  })
+
+  test('an unknown run is null rather than a throw', async () => {
+    await store.init()
+    assert.equal(await store.get('nope'), null)
+  })
+
+  test('lists newest first', async () => {
+    const first = await store.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 1,
+    })
+    const second = await store.start({
+      persona: 'raj',
+      disposition: 'realistic',
+      seed: 2,
+    })
+
+    const runs = await store.list()
+    assert.equal(runs.length, 2)
+    assert.deepEqual(
+      new Set(runs.map((run) => run.runId)),
+      new Set([first, second])
+    )
+  })
+
+  // Comparing what a persona finds this week against last week is the whole
+  // reason runs are kept.
+  test('narrows to one persona s history', async () => {
+    await store.start({ persona: 'susan', disposition: 'realistic', seed: 1 })
+    await store.start({ persona: 'raj', disposition: 'realistic', seed: 2 })
+
+    const runs = await store.list({ persona: 'susan' })
+    assert.equal(runs.length, 1)
+    assert.equal(runs[0]?.persona, 'susan')
+  })
+
+  test('completing an unknown run does not throw', async () => {
+    await store.init()
+    await store.complete('nope', {
+      findings: [],
+      tally: TALLY,
+      memory: {},
+      stoppedBy: 'exhausted',
+    })
+  })
+
+  // Not every project installs SerializePlugin, and a bare SQLite driver cannot
+  // bind a Date at all — so the whole round trip has to hold without it.
+  test('round-trips without SerializePlugin', async () => {
+    const bare = new Kysely<KyselyPikkuDB>({
+      dialect: new SqliteDialect({ database: new Database(':memory:') }),
+      plugins: [new CamelCasePlugin()],
+    })
+    const bareStore = new KyselyVirtualUserRunStore(bare)
+
+    const runId = await bareStore.start({
+      persona: 'susan',
+      disposition: 'realistic',
+      seed: 99,
+      goals: ['export a doc'],
+      memory: { docId: 'doc_1' },
+    })
+    await bareStore.complete(runId, {
+      findings: [FINDING],
+      tally: TALLY,
+      memory: { docId: 'doc_1' },
+      stoppedBy: 'exhausted',
+    })
+
+    const run = await bareStore.get(runId)
+    assert.equal(run?.status, 'completed')
+    assert.strictEqual(run?.seed, 99)
+    assert.deepEqual(run?.goals, ['export a doc'])
+    assert.deepEqual(run?.tally, TALLY)
+    assert.equal(run?.findings[0]?.detail, FINDING.detail)
+    assert.ok(run?.createdAt instanceof Date)
+    assert.ok(run?.finishedAt instanceof Date)
+  })
+})

package/src/kysely-virtual-user-run-store.ts

@@ -0,0 +1,149 @@
+import type { Kysely, Selectable } from 'kysely'
+import { randomUUID } from 'node:crypto'
+import type {
+  VirtualUserDisposition,
+  VirtualUserRunOutcome,
+  VirtualUserRunRecord,
+  VirtualUserRunStart,
+  VirtualUserRunStore,
+} from '@pikku/core/virtual-user'
+import type { KyselyPikkuDB, VirtualUserRunTable } from './kysely-tables.js'
+import { parseJson } from './kysely-json.js'
+import { ensurePikkuSchema } from './schema/index.js'
+import { virtualUserSchema } from './schema/virtual-user.schema.js'
+
+/**
+ * Records virtual-user runs in a `virtualUserRun` table.
+ *
+ * The store is the whole record of a run: `runVirtualUser` returns as soon as
+ * the row exists and the engine keeps going without it, so nothing else knows
+ * the run happened. That is why `fail()` exists as its own call — a run that
+ * crashed and a run that found nothing are different answers, and a row left at
+ * `running` is neither.
+ *
+ * Works with or without `SerializePlugin`, which not every project installs: the
+ * JSON columns are written as strings (which the plugin passes through) and read
+ * back through `parseJson` (which passes through what the plugin already
+ * parsed). Timestamps are written as ISO strings for the same reason — a bare
+ * SQLite driver cannot bind a `Date` at all.
+ *
+ * SECURITY: an `adversarial` run's findings are working exploits carrying live
+ * ids. This is a privileged store; the scaffolded reads are scope-gated, and
+ * exposing these rows more widely publishes your own exploits.
+ */
+export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
+  private initialized = false
+
+  constructor(private db: Kysely<KyselyPikkuDB>) {}
+
+  /**
+   * Creates the table on first use, like the audit sink — the runtime does not
+   * need it, so it arrives with the feature that fills it rather than in every
+   * database.
+   */
+  public async init(): Promise<void> {
+    if (this.initialized) return
+    await ensurePikkuSchema(this.db, virtualUserSchema)
+    this.initialized = true
+  }
+
+  async start(run: VirtualUserRunStart): Promise<string> {
+    await this.init()
+    const runId = randomUUID()
+    await this.db
+      .insertInto('virtualUserRun')
+      .values({
+        runId,
+        persona: run.persona,
+        disposition: run.disposition,
+        // Text on every engine (see the schema), so the seed round-trips
+        // through drivers that hand a BIGINT back as a string.
+        seed: String(run.seed),
+        status: 'running',
+        goals: JSON.stringify(run.goals ?? []),
+        memory: JSON.stringify(run.memory ?? {}),
+        findings: '[]',
+        startedBy: run.startedBy ?? null,
+      })
+      .execute()
+    return runId
+  }
+
+  async complete(runId: string, outcome: VirtualUserRunOutcome): Promise<void> {
+    await this.init()
+    await this.db
+      .updateTable('virtualUserRun')
+      .set({
+        status: 'completed',
+        findings: JSON.stringify(outcome.findings),
+        tally: JSON.stringify(outcome.tally),
+        // Overwritten rather than merged: the engine's memory already carries
+        // what it was given, plus what it learned on the way.
+        memory: JSON.stringify(outcome.memory),
+        stoppedBy: outcome.stoppedBy,
+        finishedAt: new Date().toISOString(),
+      })
+      .where('runId', '=', runId)
+      .execute()
+  }
+
+  async fail(runId: string, error: string): Promise<void> {
+    await this.init()
+    await this.db
+      .updateTable('virtualUserRun')
+      .set({
+        status: 'failed',
+        error,
+        finishedAt: new Date().toISOString(),
+      })
+      .where('runId', '=', runId)
+      .execute()
+  }
+
+  async get(runId: string): Promise<VirtualUserRunRecord | null> {
+    await this.init()
+    const row = await this.db
+      .selectFrom('virtualUserRun')
+      .selectAll()
+      .where('runId', '=', runId)
+      .executeTakeFirst()
+    return row ? this.toRecord(row) : null
+  }
+
+  async list(options?: {
+    persona?: string
+    limit?: number
+    offset?: number
+  }): Promise<VirtualUserRunRecord[]> {
+    await this.init()
+    let query = this.db.selectFrom('virtualUserRun').selectAll()
+    if (options?.persona) {
+      query = query.where('persona', '=', options.persona)
+    }
+    const rows = await query
+      .orderBy('createdAt', 'desc')
+      .limit(options?.limit ?? 50)
+      .offset(options?.offset ?? 0)
+      .execute()
+    return rows.map((row) => this.toRecord(row))
+  }
+
+  private toRecord(row: Selectable<VirtualUserRunTable>): VirtualUserRunRecord {
+    return {
+      runId: row.runId,
+      persona: row.persona,
+      disposition: row.disposition as VirtualUserDisposition,
+      seed: Number(row.seed),
+      status: row.status,
+      goals: parseJson(row.goals) ?? [],
+      memory: parseJson(row.memory) ?? {},
+      findings: parseJson(row.findings) ?? [],
+      tally: row.tally ? (parseJson(row.tally) ?? null) : null,
+      stoppedBy: row.stoppedBy,
+      error: row.error,
+      startedBy: row.startedBy,
+      createdAt: new Date(row.createdAt),
+      finishedAt: row.finishedAt ? new Date(row.finishedAt) : null,
+    }
+  }
+}

package/src/kysely-workflow-service.ts

@@ -600,6 +600,45 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       .execute()
   }
 
+  protected async findStalledRunIds(
+    before: Date,
+    limit: number
+  ): Promise<string[]> {
+    const rows = await this.db
+      .selectFrom('workflowRuns as r')
+      .select('r.workflowRunId')
+      .where('r.status', '=', 'running')
+      .where(({ not, exists, selectFrom }) =>
+        not(
+          exists(
+            selectFrom('workflowStep as s')
+              .select('s.workflowStepId')
+              .whereRef('s.workflowRunId', '=', 'r.workflowRunId')
+              .where('s.status', 'in', ['running', 'scheduled', 'suspended'])
+          )
+        )
+      )
+      // The run row only moves on a status change, so idleness has to hold for
+      // the steps too — expressed as "no step touched since `before`" rather
+      // than max(updated_at) so it stays one index-friendly NOT EXISTS.
+      .where('r.updatedAt', '<', before)
+      .where(({ not, exists, selectFrom }) =>
+        not(
+          exists(
+            selectFrom('workflowStep as s')
+              .select('s.workflowStepId')
+              .whereRef('s.workflowRunId', '=', 'r.workflowRunId')
+              .where('s.updatedAt', '>=', before)
+          )
+        )
+      )
+      .orderBy('r.updatedAt', 'asc')
+      .limit(limit)
+      .execute()
+
+    return rows.map((row) => row.workflowRunId)
+  }
+
   async getRunState(runId: string): Promise<Record<string, unknown>> {
     const row = await this.db
       .selectFrom('workflowRuns')

package/src/schema/audit.schema.ts

@@ -0,0 +1,55 @@
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * The `audit` table {@link KyselyAuditService} writes to and reads back.
+ *
+ * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
+ * a project that has wired a durable audit sink does, and creating it for
+ * everyone would put an empty table in every database. `KyselyAuditService.init()`
+ * applies it, so it arrives exactly when the sink that fills it does.
+ *
+ * Every column is text on every engine, matching the platform audit-queue
+ * consumer's row shape — a locally-run project and a deployed stage write rows
+ * the same reader can read. `occurredAt` is an ISO 8601 string rather than a
+ * timestamp for that reason: string ordering is chronological ordering, and the
+ * queue consumer has no shared type to agree on.
+ */
+export const auditSchema: PikkuSchema = {
+  name: 'audit',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('audit')
+        .addColumn('auditId', 'text', (col) => col.primaryKey())
+        .addColumn('occurredAt', 'text', (col) => col.notNull())
+        .addColumn('type', 'text', (col) => col.notNull())
+        .addColumn('source', 'text', (col) => col.defaultTo('auto').notNull())
+        .addColumn('outcome', 'text')
+        .addColumn('functionId', 'text')
+        .addColumn('wireType', 'text')
+        .addColumn('traceId', 'text')
+        .addColumn('transactionId', 'text')
+        .addColumn('queryId', 'text')
+        .addColumn('userId', 'text')
+        .addColumn('orgId', 'text')
+        .addColumn('pikkuUserId', 'text')
+        .addColumn('tables', 'text')
+        .addColumn('changedCols', 'text')
+        .addColumn('event', 'text')
+        .addColumn('old', 'text')
+        .addColumn('data', 'text'),
+
+    // The trail is only ever read newest-first, and the two filters the console
+    // offers are user and type.
+    (db) =>
+      db.schema
+        .createIndex('idx_audit_occurred_at')
+        .on('audit')
+        .column('occurredAt'),
+
+    (db) =>
+      db.schema.createIndex('idx_audit_user_id').on('audit').column('userId'),
+
+    (db) => db.schema.createIndex('idx_audit_type').on('audit').column('type'),
+  ],
+}

package/src/schema/credential.schema.ts

@@ -7,8 +7,9 @@ import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
  * The uniqueness rule is an expression index — `user_id` is nullable and two
  * rows with a null user must still collide on name — which the schema builder
  * cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
- * which is why the table is named explicitly rather than left to the search
- * path.
+ * which is why the table goes through `ctx.table` rather than being left to the
+ * search path. The index's own name stays unqualified: an index belongs to the
+ * schema of the table it is on, and postgres rejects one that says otherwise.
  */
 export const credentialSchema: PikkuSchema = {
   name: 'credential',
@@ -29,7 +30,8 @@ export const credentialSchema: PikkuSchema = {
         ),
 
     rawStatement(
-      sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`
+      ({ table }) =>
+        sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON ${table('credentials')} (name, COALESCE(user_id, ''))`
     ),
 
     (db) =>