@pikku/kysely 0.13.2 → 0.13.4

package/src/kysely-tables.test.ts

@@ -0,0 +1,110 @@
+import { test, describe } from 'node:test'
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { Kysely, SqliteDialect } from 'kysely'
+import Database from 'better-sqlite3'
+
+import { applyPikkuSchemas } from './schema/index.js'
+
+/**
+ * `KyselyPikkuDB` and the schema declaration describe the same tables, and
+ * nothing but this makes them agree.
+ *
+ * The types stay hand-written on purpose: they carry knowledge introspection
+ * cannot recover — `WorkflowStatus` rather than `string`, `Generated<Date>`
+ * rather than `Date`, and the comments explaining what a column holds.
+ * Generating them from the DDL would trade all of that for one fewer file.
+ *
+ * What generating would have bought is drift protection, and that is what these
+ * tests buy instead: add a column to a schema and forget the interface, or the
+ * reverse, and the names stop matching here.
+ */
+
+const SOURCE = readFileSync(
+  join(import.meta.dirname, 'kysely-tables.ts'),
+  'utf8'
+)
+
+/** The same transform `CamelCasePlugin` applies to every declared identifier. */
+const physical = (name: string): string =>
+  name.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()
+
+const interfaceBody = (name: string): string => {
+  const start = SOURCE.indexOf(`export interface ${name} {`)
+  assert.notEqual(start, -1, `no interface ${name}`)
+  const end = SOURCE.indexOf('\n}', start)
+  return SOURCE.slice(SOURCE.indexOf('{', start) + 1, end)
+}
+
+const fieldsOf = (name: string): string[] =>
+  interfaceBody(name)
+    .split('\n')
+    .map((line) => /^ {2}(\w+)(\??):/.exec(line)?.[1])
+    .filter((field): field is string => field !== undefined)
+
+/** `KyselyPikkuDB`'s own table-name → interface-name mapping. */
+const tableInterfaces = new Map(
+  interfaceBody('KyselyPikkuDB')
+    .split('\n')
+    .map((line) => /^ {2}(\w+): (\w+)$/.exec(line))
+    .filter((match): match is RegExpExecArray => match !== null)
+    .map((match) => [match[1]!, match[2]!])
+)
+
+/** The schema as declared, materialized and read back. */
+const declaredSchema = async (): Promise<Map<string, Set<string>>> => {
+  const db = new Kysely<any>({
+    dialect: new SqliteDialect({ database: new Database(':memory:') }),
+  })
+  try {
+    // The scope schema's grants reference it, so it has to be there first —
+    // the same prerequisite a real project satisfies with Better Auth.
+    await db.schema
+      .createTable('user')
+      .addColumn('id', 'text', (col) => col.primaryKey())
+      .execute()
+    await applyPikkuSchemas(db)
+
+    const tables = await db.introspection.getTables()
+    return new Map(
+      tables
+        .filter((table) => table.name !== 'user')
+        .map((table) => [
+          table.name,
+          new Set(table.columns.map((column) => column.name)),
+        ])
+    )
+  } finally {
+    await db.destroy()
+  }
+}
+
+describe('KyselyPikkuDB tracks the schema declaration', () => {
+  test('names every table the declaration creates, and no others', async () => {
+    const declared = await declaredSchema()
+
+    const typed = new Set([...tableInterfaces.keys()].map(physical))
+    assert.deepEqual(
+      [...typed].sort(),
+      [...declared.keys()].sort(),
+      'a table in one and not the other means the two have drifted apart'
+    )
+  })
+
+  test('names every column the declaration creates, and no others', async () => {
+    const declared = await declaredSchema()
+
+    for (const [key, interfaceName] of tableInterfaces) {
+      const table = physical(key)
+      const columns = declared.get(table)
+      assert.ok(columns, `${table} is declared nowhere`)
+
+      assert.deepEqual(
+        fieldsOf(interfaceName).map(physical).sort(),
+        [...columns].sort(),
+        `${interfaceName} and the '${table}' declaration disagree`
+      )
+    }
+  })
+})

package/src/kysely-tables.ts

@@ -127,6 +127,8 @@ export interface AIRunTable {
   errorMessage: string | null
   suspendReason: 'approval' | 'credential' | 'rpc-missing' | null
   missingRpcs: string | null
+  /** JSON-serialized approvals the run is suspended on. */
+  pendingApprovals: string | null
   usageInputTokens: Generated<number>
   usageOutputTokens: Generated<number>
   usageModel: Generated<string>

package/src/kysely-webhook-service.ts

@@ -9,8 +9,9 @@ import {
 } from '@pikku/core/services'
 import type { QueueService } from '@pikku/core/queue'
 import type { Kysely } from 'kysely'
-import { sql } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
+import { ensurePikkuSchema } from './schema/index.js'
+import { webhookSchema } from './schema/webhook.schema.js'
 
 /**
  * Durable {@link QueueWebhookService}: still delivers through the
@@ -30,59 +31,8 @@ export class KyselyWebhookService extends QueueWebhookService {
   }
 
   public async init(): Promise<void> {
-    if (this.initialized) {
-      return
-    }
-
-    await this.db.schema
-      .createTable('webhook_delivery')
-      .ifNotExists()
-      .addColumn('delivery_id', 'text', (col) => col.primaryKey())
-      .addColumn('organization_id', 'text')
-      .addColumn('url', 'text', (col) => col.notNull())
-      .addColumn('event', 'text')
-      .addColumn('status', 'text', (col) => col.defaultTo('pending').notNull())
-      .addColumn('attempts', 'integer', (col) => col.defaultTo(0).notNull())
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('updated_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('delivered_at', 'timestamp')
-      .execute()
-
-    await this.db.schema
-      .createTable('webhook_delivery_attempt')
-      .ifNotExists()
-      .addColumn('attempt_id', 'text', (col) => col.primaryKey())
-      .addColumn('delivery_id', 'text', (col) =>
-        col.notNull().references('webhook_delivery.delivery_id').onDelete('cascade')
-      )
-      .addColumn('attempt_number', 'integer', (col) => col.notNull())
-      .addColumn('status_code', 'integer')
-      .addColumn('response_body', 'text')
-      .addColumn('error', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .execute()
-
-    await this.createIndexSafe(
-      this.db.schema
-        .createIndex('idx_webhook_delivery_org')
-        .ifNotExists()
-        .on('webhook_delivery')
-        .column('organization_id')
-    )
-    await this.createIndexSafe(
-      this.db.schema
-        .createIndex('idx_webhook_delivery_attempt_delivery')
-        .ifNotExists()
-        .on('webhook_delivery_attempt')
-        .column('delivery_id')
-    )
-
+    if (this.initialized) return
+    await ensurePikkuSchema(this.db, webhookSchema)
     this.initialized = true
   }
 
@@ -184,17 +134,4 @@ export class KyselyWebhookService extends QueueWebhookService {
       .execute()
     return { delivery, attempts }
   }
-
-  private async createIndexSafe(builder: {
-    execute(): Promise<void>
-  }): Promise<void> {
-    try {
-      await builder.execute()
-    } catch (e: any) {
-      if (e?.code === 'ER_DUP_KEYNAME' || e?.errno === 1061) return
-      if (e?.code === '42P07') return
-      if (e?.message?.includes('already exists')) return
-      throw e
-    }
-  }
 }

package/src/kysely-workflow-mirror.test.ts

@@ -258,12 +258,12 @@ describe('KyselyWorkflowMirror — init', () => {
     // No-op on second call; just shouldn't throw.
   })
 
-  test('cohabits with KyselyWorkflowService schema (ifNotExists)', async () => {
-    // Same DB, schema already created by mirror.init() in beforeEach.
-    // If KyselyWorkflowService.init() ran on the same DB, both should
-    // succeed because both use ifNotExists. We can't import it without
-    // the singleton-services bootstrap, so simulate by re-calling our
-    // own init — proves the SQL is idempotent.
-    await assert.doesNotReject(mirror.init())
+  test('cohabits with KyselyWorkflowService, which declares the same tables', async () => {
+    // Both services own the workflow schema, so both call init() against the
+    // same database. The second one to run must find the tables and stop, not
+    // try to create them again. Importing the other service needs the
+    // singleton-services bootstrap, so a fresh instance stands in for it.
+    const second = new KyselyWorkflowMirror(db)
+    await assert.doesNotReject(second.init())
   })
 })

package/src/kysely-workflow-mirror.ts

@@ -8,8 +8,9 @@ import type {
   WorkflowVersionStatus,
 } from '@pikku/core/workflow'
 import type { Kysely } from 'kysely'
-import { sql } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
+import { ensurePikkuSchema } from './schema/index.js'
+import { workflowSchema } from './schema/workflow.schema.js'
 
 /**
  * Kysely-backed `WorkflowRunMirror`.
@@ -29,117 +30,17 @@ export class KyselyWorkflowMirror implements WorkflowRunMirror {
   constructor(protected db: Kysely<KyselyPikkuDB>) {}
 
   /**
-   * Create the underlying tables if they don't exist. Safe to call from
-   * either the mirror or `KyselyWorkflowService.init()` — schema is the
-   * same and uses `ifNotExists`.
+   * Create the underlying tables if none of them exist yet.
+   *
+   * Safe to call from either the mirror or `KyselyWorkflowService.init()` —
+   * both apply the same `workflowSchema`, and `ensurePikkuSchema` is a no-op
+   * once the tables are there. A database holding only *part* of the schema
+   * throws rather than filling in the rest: something else owns those tables,
+   * and boot is not where that gets reconciled.
    */
   public async init(): Promise<void> {
     if (this.initialized) return
-
-    await this.db.schema
-      .createTable('workflow_runs')
-      .ifNotExists()
-      .addColumn('workflow_run_id', 'text', (col) => col.primaryKey())
-      .addColumn('workflow', 'text', (col) => col.notNull())
-      .addColumn('status', 'text', (col) => col.notNull())
-      .addColumn('input', 'text', (col) => col.notNull())
-      .addColumn('output', 'text')
-      .addColumn('error', 'text')
-      .addColumn('state', 'text', (col) => col.defaultTo('{}'))
-      .addColumn('inline', 'boolean', (col) => col.defaultTo(false))
-      .addColumn('graph_hash', 'text')
-      .addColumn('deterministic', 'boolean', (col) => col.defaultTo(false))
-      .addColumn('planned_steps', 'text')
-      .addColumn('wire', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('updated_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .execute()
-
-    await this.db.schema
-      .createTable('workflow_step')
-      .ifNotExists()
-      .addColumn('workflow_step_id', 'text', (col) => col.primaryKey())
-      .addColumn('workflow_run_id', 'text', (col) =>
-        col
-          .notNull()
-          .references('workflow_runs.workflow_run_id')
-          .onDelete('cascade')
-      )
-      .addColumn('step_name', 'text', (col) => col.notNull())
-      .addColumn('rpc_name', 'text')
-      .addColumn('data', 'text')
-      .addColumn('status', 'text', (col) => col.notNull().defaultTo('pending'))
-      .addColumn('result', 'text')
-      .addColumn('error', 'text')
-      .addColumn('child_run_id', 'text')
-      .addColumn('branch_taken', 'text')
-      .addColumn('retries', 'integer')
-      .addColumn('retry_delay', 'text')
-      .addColumn('from_step_name', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('updated_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addUniqueConstraint('workflow_step_run_name_unique', [
-        'workflow_run_id',
-        'step_name',
-      ])
-      .execute()
-
-    // Additive: backfill the column on mirror tables created before from_step_name.
-    await this.db.schema
-      .alterTable('workflow_step')
-      .addColumn('from_step_name', 'text')
-      .execute()
-      .catch(() => {
-        // Column already exists (fresh tables include it above) — nothing to do.
-      })
-
-    await this.db.schema
-      .createTable('workflow_step_history')
-      .ifNotExists()
-      .addColumn('history_id', 'text', (col) => col.primaryKey())
-      .addColumn('workflow_step_id', 'text', (col) =>
-        col
-          .notNull()
-          .references('workflow_step.workflow_step_id')
-          .onDelete('cascade')
-      )
-      .addColumn('status', 'text', (col) => col.notNull())
-      .addColumn('result', 'text')
-      .addColumn('error', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('running_at', 'timestamp')
-      .addColumn('scheduled_at', 'timestamp')
-      .addColumn('succeeded_at', 'timestamp')
-      .addColumn('failed_at', 'timestamp')
-      .execute()
-
-    await this.db.schema
-      .createTable('workflow_versions')
-      .ifNotExists()
-      .addColumn('workflow_name', 'text', (col) => col.notNull())
-      .addColumn('graph_hash', 'text', (col) => col.notNull())
-      .addColumn('graph', 'text', (col) => col.notNull())
-      .addColumn('source', 'text', (col) => col.notNull())
-      .addColumn('status', 'text', (col) => col.notNull().defaultTo('active'))
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addPrimaryKeyConstraint('workflow_versions_pk', [
-        'workflow_name',
-        'graph_hash',
-      ])
-      .execute()
-
+    await ensurePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 

package/src/kysely-workflow-service.ts

@@ -2,6 +2,7 @@ import type { SerializedError } from '@pikku/core'
 import {
   PikkuWorkflowService,
   type WorkflowPlannedStep,
+  type WorkflowQueueOptions,
   type WorkflowRun,
   type WorkflowRunWire,
   type StepState,
@@ -10,141 +11,27 @@ import {
   type WorkflowVersionStatus,
 } from '@pikku/core/workflow'
 import type { Kysely } from 'kysely'
-import { sql } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
 import { parseJson } from './kysely-json.js'
+import { ensurePikkuSchema } from './schema/index.js'
+import { workflowSchema } from './schema/workflow.schema.js'
 
 export class KyselyWorkflowService extends PikkuWorkflowService {
   private initialized = false
   private runService: KyselyWorkflowRunService
 
-  constructor(protected db: Kysely<KyselyPikkuDB>) {
-    super()
+  constructor(
+    protected db: Kysely<KyselyPikkuDB>,
+    options: WorkflowQueueOptions = {}
+  ) {
+    super(options)
     this.runService = new KyselyWorkflowRunService(db)
   }
 
   public async init(): Promise<void> {
-    if (this.initialized) {
-      return
-    }
-
-    await this.db.schema
-      .createTable('workflow_runs')
-      .ifNotExists()
-      .addColumn('workflow_run_id', 'text', (col) =>
-        col
-          .primaryKey()
-          .defaultTo(sql`${sql.raw("'" + crypto.randomUUID() + "'")}`)
-      )
-      .addColumn('workflow', 'text', (col) => col.notNull())
-      .addColumn('status', 'text', (col) => col.notNull())
-      .addColumn('input', 'text', (col) => col.notNull())
-      .addColumn('output', 'text')
-      .addColumn('error', 'text')
-      .addColumn('state', 'text', (col) => col.defaultTo('{}'))
-      .addColumn('inline', 'boolean', (col) => col.defaultTo(false))
-      .addColumn('graph_hash', 'text')
-      .addColumn('deterministic', 'boolean', (col) => col.defaultTo(false))
-      .addColumn('planned_steps', 'text')
-      .addColumn('wire', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('updated_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .execute()
-
-    await this.db.schema
-      .createTable('workflow_step')
-      .ifNotExists()
-      .addColumn('workflow_step_id', 'text', (col) =>
-        col
-          .primaryKey()
-          .defaultTo(sql`${sql.raw("'" + crypto.randomUUID() + "'")}`)
-      )
-      .addColumn('workflow_run_id', 'text', (col) =>
-        col
-          .notNull()
-          .references('workflow_runs.workflow_run_id')
-          .onDelete('cascade')
-      )
-      .addColumn('step_name', 'text', (col) => col.notNull())
-      .addColumn('rpc_name', 'text')
-      .addColumn('data', 'text')
-      .addColumn('status', 'text', (col) => col.notNull().defaultTo('pending'))
-      .addColumn('result', 'text')
-      .addColumn('error', 'text')
-      .addColumn('child_run_id', 'text')
-      .addColumn('branch_taken', 'text')
-      .addColumn('retries', 'integer')
-      .addColumn('retry_delay', 'text')
-      .addColumn('from_step_name', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('updated_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addUniqueConstraint('workflow_step_run_name_unique', [
-        'workflow_run_id',
-        'step_name',
-      ])
-      .execute()
-
-    // Additive: backfill the column on tables created before from_step_name.
-    await this.db.schema
-      .alterTable('workflow_step')
-      .addColumn('from_step_name', 'text')
-      .execute()
-      .catch(() => {
-        // Column already exists (fresh tables include it above) — nothing to do.
-      })
-
-    await this.db.schema
-      .createTable('workflow_step_history')
-      .ifNotExists()
-      .addColumn('history_id', 'text', (col) =>
-        col
-          .primaryKey()
-          .defaultTo(sql`${sql.raw("'" + crypto.randomUUID() + "'")}`)
-      )
-      .addColumn('workflow_step_id', 'text', (col) =>
-        col
-          .notNull()
-          .references('workflow_step.workflow_step_id')
-          .onDelete('cascade')
-      )
-      .addColumn('status', 'text', (col) => col.notNull())
-      .addColumn('result', 'text')
-      .addColumn('error', 'text')
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addColumn('running_at', 'timestamp')
-      .addColumn('scheduled_at', 'timestamp')
-      .addColumn('succeeded_at', 'timestamp')
-      .addColumn('failed_at', 'timestamp')
-      .execute()
-
-    await this.db.schema
-      .createTable('workflow_versions')
-      .ifNotExists()
-      .addColumn('workflow_name', 'text', (col) => col.notNull())
-      .addColumn('graph_hash', 'text', (col) => col.notNull())
-      .addColumn('graph', 'text', (col) => col.notNull())
-      .addColumn('source', 'text', (col) => col.notNull())
-      .addColumn('status', 'text', (col) => col.notNull().defaultTo('active'))
-      .addColumn('created_at', 'timestamp', (col) =>
-        col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
-      )
-      .addPrimaryKeyConstraint('workflow_versions_pk', [
-        'workflow_name',
-        'graph_hash',
-      ])
-      .execute()
-
+    if (this.initialized) return
+    await ensurePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 
@@ -417,7 +304,11 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     if (latestHistory) {
       await this.db
         .updateTable('workflowStepHistory')
-        .set({ status: 'succeeded', result: resultJson, succeededAt: new Date() })
+        .set({
+          status: 'succeeded',
+          result: resultJson,
+          succeededAt: new Date(),
+        })
         .where('historyId', '=', latestHistory.historyId)
         .execute()
     }
@@ -586,7 +477,9 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return nodeIds.filter((id) => !existingStepNames.has(id))
   }
 
-  async getStepInstances(runId: string): Promise<
+  async getStepInstances(
+    runId: string
+  ): Promise<
     Array<{ stepName: string; status: StepStatus; fromStepName?: string }>
   > {
     const rows = await this.db