@pikku/kysely 0.13.18 → 0.13.20

package/src/kysely-tables.ts

@@ -324,6 +324,34 @@ export interface VirtualUserRunTable {
   finishedAt: Date | string | null
 }
 
+/**
+ * One audit event. Every column is text on every engine so a locally-run
+ * project and a deployed stage write rows the same reader can read; the JSON
+ * columns (`tables`, `changedCols`, `event`, `old`, `data`) are serialised by
+ * {@link KyselyAuditService} rather than by a plugin.
+ */
+export interface AuditTable {
+  auditId: string
+  /** ISO 8601, because string ordering is chronological ordering. */
+  occurredAt: string
+  type: string
+  source: Generated<string>
+  outcome: string | null
+  functionId: string | null
+  wireType: string | null
+  traceId: string | null
+  transactionId: string | null
+  queryId: string | null
+  userId: string | null
+  orgId: string | null
+  pikkuUserId: string | null
+  tables: string | null
+  changedCols: string | null
+  event: string | null
+  old: string | null
+  data: string | null
+}
+
 export interface KyselyPikkuDB {
   pikkuScopes: PikkuScopesTable
   pikkuRoles: PikkuRolesTable
@@ -354,4 +382,5 @@ export interface KyselyPikkuDB {
   webhookDelivery: WebhookDeliveryTable
   webhookDeliveryAttempt: WebhookDeliveryAttemptTable
   virtualUserRun: VirtualUserRunTable
+  audit: AuditTable
 }

package/src/kysely-virtual-user-run-store.test.ts

@@ -6,6 +6,7 @@ import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
 import { KyselyVirtualUserRunStore } from './kysely-virtual-user-run-store.js'
 import { SerializePlugin } from './serialize-plugin.js'
 import type { KyselyPikkuDB } from './kysely-tables.js'
+import { applyPikkuSchemas, virtualUserSchema } from './schema/index.js'
 
 // SerializePlugin is here because most projects in this package run with it, and
 // it is the harder case: it deserialises JSON columns the store has already
@@ -16,6 +17,12 @@ const newDb = () =>
     plugins: [new CamelCasePlugin(), new SerializePlugin()],
   })
 
+const migratedDb = async () => {
+  const db = newDb()
+  await applyPikkuSchemas(db, [virtualUserSchema])
+  return db
+}
+
 const TALLY = {
   steps: 12,
   calls: 30,
@@ -36,15 +43,23 @@ describe('KyselyVirtualUserRunStore', () => {
   let db: Kysely<KyselyPikkuDB>
   let store: KyselyVirtualUserRunStore
 
-  beforeEach(() => {
-    db = newDb()
+  beforeEach(async () => {
+    db = await migratedDb()
     store = new KyselyVirtualUserRunStore(db)
   })
 
-  // The runtime never needs this table, so it arrives with the feature that
-  // fills it rather than in every database — which means the first write has to
-  // create it.
-  test('creates its table on first use', async () => {
+  // The runtime never creates this table, so a project that skipped the
+  // migration has to be told which command writes it rather than quietly
+  // getting one at first use.
+  test('refuses to start against a database that never migrated it', async () => {
+    const store = new KyselyVirtualUserRunStore(newDb())
+    await assert.rejects(
+      store.start({ persona: 'susan', disposition: 'realistic', seed: 42 }),
+      /pikku db generate/
+    )
+  })
+
+  test('writes against a migrated table', async () => {
     const runId = await store.start({
       persona: 'susan',
       disposition: 'realistic',
@@ -184,6 +199,7 @@ describe('KyselyVirtualUserRunStore', () => {
       dialect: new SqliteDialect({ database: new Database(':memory:') }),
       plugins: [new CamelCasePlugin()],
     })
+    await applyPikkuSchemas(bare, [virtualUserSchema])
     const bareStore = new KyselyVirtualUserRunStore(bare)
 
     const runId = await bareStore.start({

package/src/kysely-virtual-user-run-store.ts

@@ -9,7 +9,7 @@ import type {
 } from '@pikku/core/virtual-user'
 import type { KyselyPikkuDB, VirtualUserRunTable } from './kysely-tables.js'
 import { parseJson } from './kysely-json.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { virtualUserSchema } from './schema/virtual-user.schema.js'
 
 /**
@@ -43,7 +43,7 @@ export class KyselyVirtualUserRunStore implements VirtualUserRunStore {
    */
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, virtualUserSchema)
+    await requirePikkuSchema(this.db, virtualUserSchema)
     this.initialized = true
   }
 

package/src/kysely-webhook-service.test.ts

@@ -7,6 +7,7 @@ import { setSingletonServices } from '@pikku/core/state'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { SerializePlugin } from './serialize-plugin.js'
 import { KyselyWebhookService } from './kysely-webhook-service.js'
+import { applyPikkuSchemas, webhookSchema } from './schema/index.js'
 
 const noopLogger = { error() {}, info() {}, warn() {}, debug() {} }
 
@@ -50,6 +51,7 @@ describe('KyselyWebhookService', () => {
   test('send() inserts a pending delivery and enqueues with deliveryId === jobId', async () => {
     const db = createDb()
     const { added, queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 
@@ -76,6 +78,7 @@ describe('KyselyWebhookService', () => {
   test('recordAttempt() appends attempts and rolls status forward', async () => {
     const db = createDb()
     const { queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 
@@ -104,6 +107,7 @@ describe('KyselyWebhookService', () => {
   test('listDeliveries() filters by organization', async () => {
     const db = createDb()
     const { queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 

package/src/kysely-webhook-service.ts

@@ -10,7 +10,7 @@ import {
 } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { webhookSchema } from './schema/webhook.schema.js'
 
 /**
@@ -32,7 +32,7 @@ export class KyselyWebhookService extends QueueWebhookService {
 
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, webhookSchema)
+    await requirePikkuSchema(this.db, webhookSchema)
     this.initialized = true
   }
 

package/src/kysely-workflow-mirror.test.ts

@@ -7,6 +7,7 @@ import Database from 'better-sqlite3'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowMirror } from './kysely-workflow-mirror.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
+import { applyPikkuSchemas, workflowSchema } from './schema/index.js'
 
 let db: Kysely<KyselyPikkuDB>
 let mirror: KyselyWorkflowMirror
@@ -20,6 +21,7 @@ beforeEach(async () => {
     dialect: new SqliteDialect({ database: new Database(':memory:') }),
     plugins: [new CamelCasePlugin(), new SerializePlugin()],
   })
+  await applyPikkuSchemas(db, [workflowSchema])
   mirror = new KyselyWorkflowMirror(db)
   await mirror.init()
   runService = new KyselyWorkflowRunService(db)

package/src/kysely-workflow-mirror.ts

@@ -9,7 +9,7 @@ import type {
 } from '@pikku/core/workflow'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { workflowSchema } from './schema/workflow.schema.js'
 
 /**
@@ -33,14 +33,14 @@ export class KyselyWorkflowMirror implements WorkflowRunMirror {
    * Create the underlying tables if none of them exist yet.
    *
    * Safe to call from either the mirror or `KyselyWorkflowService.init()` —
-   * both apply the same `workflowSchema`, and `ensurePikkuSchema` is a no-op
+   * both apply the same `workflowSchema`, and `requirePikkuSchema` is a no-op
    * once the tables are there. A database holding only *part* of the schema
    * throws rather than filling in the rest: something else owns those tables,
    * and boot is not where that gets reconciled.
    */
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, workflowSchema)
+    await requirePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 

package/src/kysely-workflow-service.test.ts

@@ -3,9 +3,11 @@ import assert from 'node:assert/strict'
 import { CamelCasePlugin, Kysely, SqliteDialect, sql } from 'kysely'
 import Database from 'better-sqlite3'
 
+import type { StepState } from '@pikku/core/ecosystem/workflow'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { SerializePlugin } from './serialize-plugin.js'
 import { KyselyWorkflowService } from './kysely-workflow-service.js'
+import { applyPikkuSchemas, workflowSchema } from './schema/index.js'
 
 let db: Kysely<KyselyPikkuDB>
 let service: KyselyWorkflowService
@@ -30,6 +32,7 @@ const createDb = () => {
 
 beforeEach(async () => {
   db = createDb()
+  await applyPikkuSchemas(db, [workflowSchema])
   service = new KyselyWorkflowService(db, { wireQueues: false } as any)
   await service.init()
 })
@@ -428,3 +431,114 @@ describe('KyselyWorkflowService — a transition that addresses no history row',
     assert.equal(history[0]!.status, 'succeeded')
   })
 })
+
+/**
+ * Dispatch is at-least-once — the relay re-dispatches a step it believes was
+ * dropped, and a queue can redeliver a job it already handed out — so the claim
+ * is the only thing standing between a duplicate dispatch and a second
+ * execution of a side-effecting step.
+ */
+describe('KyselyWorkflowService — claiming a step for execution', () => {
+  const claim = (runId: string, stepName: string, rpcName = 'rpc.fn') =>
+    (service as any).claimStepForExecution(
+      runId,
+      stepName,
+      rpcName
+    ) as Promise<StepState | null>
+
+  test('two dispatches racing for the same pending step: exactly one wins', async () => {
+    const runId = await seedRun()
+    await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+
+    const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
+    const winners = claims.filter((c) => c !== null)
+
+    assert.equal(
+      winners.length,
+      1,
+      `both dispatches claimed the step, so a side-effecting step would run twice: ${JSON.stringify(claims)}`
+    )
+    assert.equal((await service.getStepState(runId, 's1')).status, 'running')
+  })
+
+  test('two dispatches racing to retry the same failed step: exactly one wins', async () => {
+    const runId = await seedRun()
+    const step = await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+    await service.setStepRunning(step.stepId)
+    await service.setStepError(step.stepId, new Error('boom'))
+
+    const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
+    const winners = claims.filter((c) => c !== null)
+
+    assert.equal(
+      winners.length,
+      1,
+      `both dispatches started a retry, so the failed step would be retried twice concurrently: ${JSON.stringify(claims)}`
+    )
+    assert.equal(
+      (await service.getStepState(runId, 's1')).attemptCount,
+      2,
+      'the retry raced itself and burned more than one attempt'
+    )
+  })
+
+  test('a step already claimed is not claimable again', async () => {
+    const runId = await seedRun()
+    await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+
+    assert.notEqual(await claim(runId, 's1'), null)
+    assert.equal(
+      await claim(runId, 's1'),
+      null,
+      'a redelivered job re-claimed a step that is already running'
+    )
+  })
+
+  test('a succeeded step is not claimable', async () => {
+    const runId = await seedRun()
+    const step = await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+    await service.setStepRunning(step.stepId)
+    await service.setStepResult(step.stepId, { ok: true })
+
+    assert.equal(await claim(runId, 's1'), null)
+  })
+
+  test('a claim that fails releases the step, so it is retryable', async () => {
+    const runId = await seedRun()
+    const step = await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+
+    const first = await claim(runId, 's1')
+    assert.notEqual(first, null)
+    await service.setStepError(step.stepId, new Error('boom'))
+
+    const retry = await claim(runId, 's1')
+    assert.notEqual(
+      retry,
+      null,
+      'the failed step stayed claimed, so it can never be retried'
+    )
+    assert.equal(retry?.status, 'running')
+    assert.equal(retry?.attemptCount, 2)
+  })
+
+  test('a claim for a different function than the step was dispatched with is refused', async () => {
+    const runId = await seedRun()
+    await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+
+    await assert.rejects(() => claim(runId, 's1', 'rpc.other'))
+    assert.equal(
+      (await service.getStepState(runId, 's1')).status,
+      'pending',
+      'the mismatched claim moved the step anyway'
+    )
+  })
+
+  test('a scheduled step is claimable, and only once', async () => {
+    const runId = await seedRun()
+    const step = await service.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
+    await service.setStepScheduled(step.stepId)
+
+    const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
+    assert.equal(claims.filter((c) => c !== null).length, 1)
+  })
+})

package/src/kysely-workflow-service.ts

@@ -1,5 +1,8 @@
 import type { SerializedError } from '@pikku/core/errors'
-import { PikkuWorkflowService } from '@pikku/core/workflow'
+import {
+  PikkuWorkflowService,
+  WorkflowStepFunctionMismatchError,
+} from '@pikku/core/workflow'
 import type {
   WorkflowPlannedStep,
   WorkflowQueueOptions,
@@ -14,7 +17,7 @@ import { sql, type Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
 import { parseJson } from './kysely-json.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { workflowSchema } from './schema/workflow.schema.js'
 
 export class KyselyWorkflowService extends PikkuWorkflowService {
@@ -31,7 +34,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
 
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, workflowSchema)
+    await requirePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 
@@ -460,6 +463,16 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return fn()
   }
 
+  /**
+   * A pass-through, and deliberately so: the one decision that must exclude —
+   * claiming a step to execute it — is made by `claimStepForExecution` below as
+   * a single conditional write, which needs no lock to be exclusive.
+   *
+   * What is left are the suspend and approval sections in the engine, where the
+   * lock is genuine mutual exclusion rather than a claim. A dialect with a real
+   * primitive overrides this to cover them — `kysely-postgres` with
+   * `pg_advisory_xact_lock`, `kysely-mysql` with `GET_LOCK`.
+   */
   async withStepLock<T>(
     _runId: string,
     _stepName: string,
@@ -468,6 +481,79 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return fn()
   }
 
+  /**
+   * Claim the step with a status-guarded `UPDATE` and read the affected-row
+   * count: the database decides the winner in one statement, so two dispatches
+   * racing for the same step cannot both proceed.
+   *
+   * This replaces the read-then-write the base engine does under `withStepLock`,
+   * which is only as exclusive as that lock — and every dialect but Postgres and
+   * MySQL inherits a pass-through. A conditional update needs no advisory-lock
+   * primitive, so every SQL dialect gets the same guarantee.
+   *
+   * The winner then goes through the ordinary transition methods, so history
+   * rows and the mirror see exactly what they saw before.
+   */
+  protected override async claimStepForExecution(
+    runId: string,
+    stepName: string,
+    rpcName: string
+  ): Promise<StepState | null> {
+    const stepState = await this.getStepState(runId, stepName)
+
+    // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
+    if (
+      stepState.rpcName !== undefined &&
+      stepState.rpcName !== (rpcName ?? null)
+    ) {
+      throw new WorkflowStepFunctionMismatchError(runId, stepName)
+    }
+
+    if (stepState.status === 'succeeded' || stepState.status === 'running') {
+      return null
+    }
+
+    if (stepState.status === 'failed') {
+      if (!(await this.claimStepStatus(stepState.stepId, ['failed']))) {
+        return null
+      }
+      return this.createRetryAttempt(stepState.stepId, 'running')
+    }
+
+    if (stepState.status === 'pending' || stepState.status === 'scheduled') {
+      if (
+        !(await this.claimStepStatus(stepState.stepId, [
+          'pending',
+          'scheduled',
+        ]))
+      ) {
+        return null
+      }
+      await this.setStepRunning(stepState.stepId)
+      return { ...stepState, status: 'running' }
+    }
+
+    return stepState
+  }
+
+  /**
+   * Move a step to `running` only if it is still in one of `from`, reporting
+   * whether this caller is the one that moved it.
+   */
+  private async claimStepStatus(
+    stepId: string,
+    from: StepStatus[]
+  ): Promise<boolean> {
+    const claimed = await this.db
+      .updateTable('workflowStep')
+      .set({ status: 'running', updatedAt: new Date() })
+      .where('workflowStepId', '=', stepId)
+      .where('status', 'in', from)
+      .executeTakeFirst()
+
+    return Number(claimed?.numUpdatedRows ?? 0n) > 0
+  }
+
   async getCompletedGraphState(runId: string): Promise<{
     completedNodeIds: string[]
     failedNodeIds: string[]
@@ -606,16 +692,18 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
   }
 
   /**
-   * The undispatched-step query, shared by the dialects that can safely relay.
+   * The undispatched-step query, shared by every dialect.
    *
-   * Deliberately NOT an override of `findUndispatchedSteps`: this class's
-   * `withStepLock` is a pass-through, so a subclass inheriting it (kysely-sqlite)
-   * has no atomic claim, and the relay's redundant dispatches would become
-   * double executions. A dialect opts in by overriding `findUndispatchedSteps`
-   * to call this — which `kysely-postgres` and `kysely-mysql` do, having real
-   * locks.
+   * It used to be deliberately NOT an override, because this class's
+   * `withStepLock` is a pass-through: a subclass inheriting it (kysely-sqlite)
+   * had no atomic claim, so the relay's redundant dispatches would have become
+   * double executions, and only `kysely-postgres` and `kysely-mysql` opted in
+   * on the strength of their real locks. `claimStepForExecution` no longer
+   * needs that lock — the claim is a status-guarded `UPDATE`, atomic in every
+   * dialect — so the relay is safe here for all of them and the override lives
+   * in the base.
    */
-  protected async queryUndispatchedSteps(
+  protected override async findUndispatchedSteps(
     before: Date,
     limit: number
   ): Promise<Array<{ runId: string; stepId: string }>> {

package/src/schema/agent.schema.ts

@@ -13,6 +13,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  */
 export const agentSchema: PikkuSchema = {
   name: 'agent',
+  ownedBy: ['agentStorage', 'agentRunState', 'agentRunService'],
   statements: [
     (db) =>
       db.schema

package/src/schema/audit.schema.ts

@@ -3,10 +3,11 @@ import type { PikkuSchema } from './pikku-schema.types.js'
 /**
  * The `audit` table {@link KyselyAuditService} writes to and reads back.
  *
- * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
- * a project that has wired a durable audit sink does, and creating it for
- * everyone would put an empty table in every database. `KyselyAuditService.init()`
- * applies it, so it arrives exactly when the sink that fills it does.
+ * Gated by `ownedBy` rather than written for everyone: only a project that has
+ * wired a durable audit sink needs this table, and generating it for the rest
+ * would put an empty table in every database. `KyselyAuditService.init()`
+ * requires it and never creates it, so it arrives with the migration `pikku db
+ * generate` writes for the sink that fills it.
  *
  * Every column is text on every engine, matching the platform audit-queue
  * consumer's row shape — a locally-run project and a deployed stage write rows
@@ -16,6 +17,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  */
 export const auditSchema: PikkuSchema = {
   name: 'audit',
+  ownedBy: ['audit', 'auditLog'],
   statements: [
     (db) =>
       db.schema

package/src/schema/channel.schema.ts

@@ -4,6 +4,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
 /** Channels and their topic subscriptions, also read by the event-hub store. */
 export const channelSchema: PikkuSchema = {
   name: 'channel',
+  ownedBy: ['channelStore', 'eventHub'],
   statements: [
     (db) =>
       db.schema

package/src/schema/credential.schema.ts

@@ -13,6 +13,7 @@ import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
  */
 export const credentialSchema: PikkuSchema = {
   name: 'credential',
+  ownedBy: ['credentialService'],
   statements: [
     (db) =>
       db.schema