@pikku/kysely 0.13.4 → 0.13.6

package/src/kysely-workflow-service.ts

@@ -10,7 +10,7 @@ import {
   type WorkflowStatus,
   type WorkflowVersionStatus,
 } from '@pikku/core/workflow'
-import type { Kysely } from 'kysely'
+import { sql, type Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
 import { parseJson } from './kysely-json.js'
@@ -112,12 +112,13 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         retries: stepOptions?.retries ?? null,
         retryDelay: stepOptions?.retryDelay?.toString() ?? null,
         fromStepName: fromStepName ?? null,
+        currentAttempt: 1,
         createdAt: now,
         updatedAt: now,
       })
       .execute()
 
-    await this.insertHistoryRecord(stepId, 'pending')
+    await this.insertHistoryRecord(stepId, 'pending', 1)
 
     return {
       stepId,
@@ -135,31 +136,24 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
 
   async getStepState(runId: string, stepName: string): Promise<StepState> {
     const row = await this.db
-      .selectFrom('workflowStep as s')
+      .selectFrom('workflowStep')
       .select([
-        's.workflowStepId',
-        's.status',
-        's.result',
-        's.error',
-        's.retries',
-        's.retryDelay',
-        's.fromStepName',
-        's.createdAt',
-        's.updatedAt',
+        'workflowStepId',
+        'status',
+        'result',
+        'error',
+        'retries',
+        'retryDelay',
+        'fromStepName',
+        // `current_attempt` is bumped alongside every history insert, so it is
+        // already the count this used to aggregate — and reading it keeps the
+        // engine's hottest query off a table that grows for the life of the run.
+        'currentAttempt',
+        'createdAt',
+        'updatedAt',
       ])
-      .select((eb) =>
-        eb
-          .selectFrom('workflowStepHistory')
-          .select(eb.fn.countAll<number>().as('cnt'))
-          .whereRef(
-            'workflowStepHistory.workflowStepId',
-            '=',
-            's.workflowStepId'
-          )
-          .as('attemptCount')
-      )
-      .where('s.workflowRunId', '=', runId)
-      .where('s.stepName', '=', stepName)
+      .where('workflowRunId', '=', runId)
+      .where('stepName', '=', stepName)
       .executeTakeFirst()
 
     if (!row) {
@@ -173,7 +167,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       status: row.status as StepState['status'],
       result: parseJson(row.result),
       error: parseJson(row.error),
-      attemptCount: Number(row.attemptCount),
+      attemptCount: Number(row.currentAttempt ?? 1),
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
       fromStepName: row.fromStepName ?? undefined,
@@ -182,6 +176,42 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     }
   }
 
+  protected override async listStepStates(
+    runId: string
+  ): Promise<Array<StepState & { stepName: string }>> {
+    const rows = await this.db
+      .selectFrom('workflowStep')
+      .select([
+        'workflowStepId',
+        'stepName',
+        'status',
+        'result',
+        'error',
+        'retries',
+        'retryDelay',
+        'fromStepName',
+        'currentAttempt',
+        'createdAt',
+        'updatedAt',
+      ])
+      .where('workflowRunId', '=', runId)
+      .execute()
+
+    return rows.map((row) => ({
+      stepId: row.workflowStepId,
+      stepName: row.stepName,
+      status: row.status as StepState['status'],
+      result: parseJson(row.result),
+      error: parseJson(row.error),
+      attemptCount: Number(row.currentAttempt ?? 1),
+      retries: row.retries != null ? Number(row.retries) : undefined,
+      retryDelay: row.retryDelay ?? undefined,
+      fromStepName: row.fromStepName ?? undefined,
+      createdAt: new Date(row.createdAt),
+      updatedAt: new Date(row.updatedAt),
+    }))
+  }
+
   async getRunHistory(
     runId: string
   ): Promise<Array<StepState & { stepName: string }>> {
@@ -189,27 +219,9 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
   }
 
   protected async setStepRunningImpl(stepId: string): Promise<void> {
-    await this.db
-      .updateTable('workflowStep')
-      .set({ status: 'running', updatedAt: new Date() })
-      .where('workflowStepId', '=', stepId)
-      .execute()
-
-    const latestHistory = await this.db
-      .selectFrom('workflowStepHistory')
-      .select('historyId')
-      .where('workflowStepId', '=', stepId)
-      .orderBy('createdAt', 'desc')
-      .limit(1)
-      .executeTakeFirst()
-
-    if (latestHistory) {
-      await this.db
-        .updateTable('workflowStepHistory')
-        .set({ status: 'running', runningAt: new Date() })
-        .where('historyId', '=', latestHistory.historyId)
-        .execute()
-    }
+    await this.writeStepTransition(stepId, 'running', {
+      runningAt: new Date(),
+    })
   }
 
   protected async setStepScheduledImpl(stepId: string): Promise<void> {
@@ -220,9 +232,88 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       .execute()
   }
 
+  /**
+   * Move a step and its current history attempt to the same status in one
+   * transaction.
+   *
+   * Both halves matter: a crash between them used to leave the step row saying
+   * `succeeded` while its history row still said `running`, which silently
+   * corrupts `getRunHistory` and every timeline reconstruction built on it.
+   * The history row is targeted by its `attempt` rather than by a `LIMIT 1`
+   * over `created_at`, so a retry that lands in the same millisecond as the
+   * attempt it replaces still resolves the newer row.
+   */
+  private async writeStepTransition(
+    stepId: string,
+    status: StepStatus,
+    historyValues: Record<string, unknown>
+  ): Promise<void> {
+    const now = new Date()
+    const stepValues: Record<string, unknown> = { status, updatedAt: now }
+    if ('result' in historyValues) stepValues.result = historyValues.result
+    if ('error' in historyValues) stepValues.error = historyValues.error
+    // A step carries only its latest outcome, so the other half is cleared.
+    if (status === 'succeeded') stepValues.error = null
+    if (status === 'failed') stepValues.result = null
+
+    await this.db.transaction().execute(async (trx) => {
+      await trx
+        .updateTable('workflowStep')
+        .set({
+          ...stepValues,
+          // A step that somehow reaches a transition without a live attempt
+          // addresses no history row at all, so give it one here. Written as an
+          // expression over the row's own column rather than a subquery, which
+          // MySQL would reject against the table being updated.
+          currentAttempt: sql`coalesce(current_attempt, 1)`,
+        } as any)
+        .where('workflowStepId', '=', stepId)
+        .execute()
+
+      const written = await trx
+        .updateTable('workflowStepHistory')
+        .set({ status, ...historyValues } as any)
+        .where('workflowStepId', '=', stepId)
+        // Correlate through the step row rather than a MAX() over this table:
+        // MySQL rejects a subquery that reads the table being updated, and a
+        // primary-key lookup is cheaper than an aggregate anyway.
+        .where('attempt', '=', (eb) =>
+          eb
+            .selectFrom('workflowStep')
+            .select('currentAttempt')
+            .where('workflowStepId', '=', stepId)
+        )
+        .executeTakeFirst()
+
+      // Matching nothing means the attempt has no history row to move, which
+      // would commit the step's new status with no record of it — precisely
+      // the divergence this transaction exists to prevent. Write the missing
+      // row instead of letting the pair drift apart.
+      if (Number(written?.numUpdatedRows ?? 0n) === 0) {
+        const step = await trx
+          .selectFrom('workflowStep')
+          .select('currentAttempt')
+          .where('workflowStepId', '=', stepId)
+          .executeTakeFirst()
+        await trx
+          .insertInto('workflowStepHistory')
+          .values({
+            historyId: crypto.randomUUID(),
+            workflowStepId: stepId,
+            status,
+            attempt: step?.currentAttempt ?? 1,
+            createdAt: now,
+            ...historyValues,
+          } as any)
+          .execute()
+      }
+    })
+  }
+
   private async insertHistoryRecord(
     stepId: string,
     status: string,
+    attempt: number,
     result?: any,
     error?: SerializedError
   ): Promise<void> {
@@ -231,6 +322,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       historyId: crypto.randomUUID(),
       workflowStepId: stepId,
       status,
+      attempt,
       result: result != null ? JSON.stringify(result) : null,
       error: error != null ? JSON.stringify(error) : null,
       createdAt: now,
@@ -280,38 +372,10 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     stepId: string,
     result: any
   ): Promise<void> {
-    const resultJson = JSON.stringify(result)
-
-    await this.db
-      .updateTable('workflowStep')
-      .set({
-        status: 'succeeded',
-        result: resultJson,
-        error: null,
-        updatedAt: new Date(),
-      })
-      .where('workflowStepId', '=', stepId)
-      .execute()
-
-    const latestHistory = await this.db
-      .selectFrom('workflowStepHistory')
-      .select('historyId')
-      .where('workflowStepId', '=', stepId)
-      .orderBy('createdAt', 'desc')
-      .limit(1)
-      .executeTakeFirst()
-
-    if (latestHistory) {
-      await this.db
-        .updateTable('workflowStepHistory')
-        .set({
-          status: 'succeeded',
-          result: resultJson,
-          succeededAt: new Date(),
-        })
-        .where('historyId', '=', latestHistory.historyId)
-        .execute()
-    }
+    await this.writeStepTransition(stepId, 'succeeded', {
+      result: JSON.stringify(result),
+      succeededAt: new Date(),
+    })
   }
 
   protected async setStepErrorImpl(
@@ -323,73 +387,54 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       stack: error.stack,
       code: (error as any).code,
     }
-    const errorJson = JSON.stringify(serializedError)
-
-    await this.db
-      .updateTable('workflowStep')
-      .set({
-        status: 'failed',
-        error: errorJson,
-        result: null,
-        updatedAt: new Date(),
-      })
-      .where('workflowStepId', '=', stepId)
-      .execute()
-
-    const latestHistory = await this.db
-      .selectFrom('workflowStepHistory')
-      .select('historyId')
-      .where('workflowStepId', '=', stepId)
-      .orderBy('createdAt', 'desc')
-      .limit(1)
-      .executeTakeFirst()
-
-    if (latestHistory) {
-      await this.db
-        .updateTable('workflowStepHistory')
-        .set({ status: 'failed', error: errorJson, failedAt: new Date() })
-        .where('historyId', '=', latestHistory.historyId)
-        .execute()
-    }
+    await this.writeStepTransition(stepId, 'failed', {
+      error: JSON.stringify(serializedError),
+      failedAt: new Date(),
+    })
   }
 
   protected async createRetryAttemptImpl(
     stepId: string,
     status: 'pending' | 'running'
   ): Promise<StepState> {
+    // Read before writing: the new attempt number comes from the history this
+    // step already has, and the row it reads is the one returned below.
+    const previous = await this.db
+      .selectFrom('workflowStep')
+      .select('currentAttempt')
+      .where('workflowStepId', '=', stepId)
+      .executeTakeFirst()
+    const attempt = (previous?.currentAttempt ?? 0) + 1
+
     await this.db
       .updateTable('workflowStep')
-      .set({ status, result: null, error: null, updatedAt: new Date() })
+      .set({
+        status,
+        result: null,
+        error: null,
+        currentAttempt: attempt,
+        updatedAt: new Date(),
+      })
       .where('workflowStepId', '=', stepId)
       .execute()
 
-    await this.insertHistoryRecord(stepId, status)
+    await this.insertHistoryRecord(stepId, status, attempt)
 
     const row = await this.db
-      .selectFrom('workflowStep as s')
+      .selectFrom('workflowStep')
       .select([
-        's.workflowStepId',
-        's.status',
-        's.result',
-        's.error',
-        's.retries',
-        's.retryDelay',
-        's.fromStepName',
-        's.createdAt',
-        's.updatedAt',
+        'workflowStepId',
+        'status',
+        'result',
+        'error',
+        'retries',
+        'retryDelay',
+        'fromStepName',
+        'currentAttempt',
+        'createdAt',
+        'updatedAt',
       ])
-      .select((eb) =>
-        eb
-          .selectFrom('workflowStepHistory')
-          .select(eb.fn.countAll<number>().as('cnt'))
-          .whereRef(
-            'workflowStepHistory.workflowStepId',
-            '=',
-            's.workflowStepId'
-          )
-          .as('attemptCount')
-      )
-      .where('s.workflowStepId', '=', stepId)
+      .where('workflowStepId', '=', stepId)
       .executeTakeFirstOrThrow()
 
     return {
@@ -397,7 +442,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       status: row.status as StepState['status'],
       result: parseJson(row.result),
       error: parseJson(row.error),
-      attemptCount: Number(row.attemptCount),
+      attemptCount: Number(row.currentAttempt ?? 1),
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
       fromStepName: row.fromStepName ?? undefined,
@@ -526,23 +571,48 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       .execute()
   }
 
+  /**
+   * Merge one key into the run's JSON state, as a single expression evaluated
+   * by the database.
+   *
+   * The SQL is dialect-specific, so subclasses override this rather than the
+   * caller. The default is the SQLite form; MySQL and Postgres differ in how
+   * they cast a JSON literal.
+   *
+   * @param path - JSON path to the key, already quoted (`$."key"`)
+   * @param json - The value as JSON text
+   */
+  protected jsonSetState(path: string, json: string) {
+    return sql<string>`json_set(coalesce(state, '{}'), ${path}, json(${json}))`
+  }
+
+  /**
+   * A JSON path for an arbitrary key. State keys are user-supplied (a graph's
+   * `setState` name, an approval's hex-encoded reason), so the key is quoted
+   * rather than interpolated bare — an unquoted `$.a.b` would silently address
+   * a nested path instead of the key literally called `a.b`.
+   */
+  private jsonPathFor(name: string): string {
+    return `$.${JSON.stringify(name)}`
+  }
+
   protected async updateRunStateImpl(
     runId: string,
     name: string,
     value: unknown
   ): Promise<void> {
-    const row = await this.db
-      .selectFrom('workflowRuns')
-      .select('state')
-      .where('workflowRunId', '=', runId)
-      .executeTakeFirst()
-
-    const state: Record<string, unknown> = parseJson(row?.state) ?? {}
-    state[name] = value
-
+    // Merged by the database, not read-modify-written in JS: two graph nodes
+    // calling setState concurrently used to race, and the later write silently
+    // dropped whichever key the earlier one had added.
     await this.db
       .updateTable('workflowRuns')
-      .set({ state: JSON.stringify(state), updatedAt: new Date() })
+      .set({
+        state: this.jsonSetState(
+          this.jsonPathFor(name),
+          JSON.stringify(value ?? null)
+        ),
+        updatedAt: new Date(),
+      })
       .where('workflowRunId', '=', runId)
       .execute()
   }
@@ -598,24 +668,5 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return this.runService.getWorkflowVersion(name, graphHash)
   }
 
-  async getAIGeneratedWorkflows(
-    agentName?: string
-  ): Promise<Array<{ workflowName: string; graphHash: string; graph: any }>> {
-    let query = this.db
-      .selectFrom('workflowVersions')
-      .select(['workflowName', 'graphHash', 'graph'])
-      .where('source', '=', 'dynamic-workflow')
-      .where('status', '=', 'active')
-    if (agentName) {
-      query = query.where('workflowName', 'like', `ai:${agentName}:%`)
-    }
-    const rows = await query.execute()
-    return rows.map((row) => ({
-      workflowName: row.workflowName,
-      graphHash: row.graphHash,
-      graph: typeof row.graph === 'string' ? JSON.parse(row.graph) : row.graph,
-    }))
-  }
-
   async close(): Promise<void> {}
 }

package/src/schema/index.ts

@@ -143,27 +143,106 @@ export const applyPikkuSchemas = async (
     const bound = bind(trx)
     for (const schema of schemas) {
       for (const statement of schema.statements) {
-        await statement(bound, types).execute()
+        const query = statement(bound, types)
+        try {
+          await query.execute()
+        } catch (error) {
+          throw explain(error, query.compile().sql, schema)
+        }
       }
     }
   })
 }
 
+/** `references "app"."workflow_runs"` — a foreign key onto a qualified table. */
+const QUALIFIED_REFERENCE = /references "[^"]+"\."[^"]+"/i
+
+/**
+ * sqlite's own words for "you put a dot where a table name goes".
+ *
+ * Matched rather than a driver's `code`, because every sqlite binding
+ * (better-sqlite3, node:sqlite, bun:sqlite, libsql) passes the engine's message
+ * through verbatim while disagreeing about how to label it. Postgres phrases the
+ * same class of error the other way round — `syntax error at or near "."` — so
+ * this cannot fire on it.
+ */
+const SQLITE_REJECTED_QUALIFIER = /near "\.":\s*syntax error/i
+
+/**
+ * Say what a schema-bound connection did to the DDL, when that is what failed.
+ *
+ * `withSchema(...)` qualifies foreign key targets along with everything else.
+ * Postgres needs that — an unqualified reference resolves against `search_path`,
+ * which a schema-bound connection does not control — and sqlite refuses it: a
+ * `REFERENCES` clause there takes a bare table name, because a foreign key can
+ * only point inside the same database, so there is no qualifier to give. One
+ * declaration cannot spell both, and what the engine says about it is
+ * `near ".": syntax error`, which names neither the schema nor the cause.
+ *
+ * Gated on sqlite's exact phrasing as well as the SQL. Postgres compiles the
+ * same qualified reference legitimately, and can raise a syntax error of its own
+ * on this DDL — a column type resolved from `requires` goes in verbatim — so a
+ * looser `/syntax error/` would answer for postgres in sqlite's name.
+ */
+const explain = (error: unknown, sql: string, schema: PikkuSchema): unknown => {
+  const message = error instanceof Error ? error.message : String(error)
+  if (
+    !QUALIFIED_REFERENCE.test(sql) ||
+    !SQLITE_REJECTED_QUALIFIER.test(message)
+  ) {
+    return error
+  }
+  return new Error(
+    `The '${schema.name}' schema has a foreign key, and this connection is bound to a schema, ` +
+      `so it compiled to a qualified 'references "schema"."table"' that the database rejected. ` +
+      'sqlite is the engine that does this: its REFERENCES clause takes a bare table name. ' +
+      'Use a connection without withSchema(...) there — sqlite has one schema to be in — ' +
+      'or write the schema down as a migration with `pikku db generate` and apply it yourself. ' +
+      `The database said: ${message}`,
+    { cause: error }
+  )
+}
+
+/** A table a schema creates, as the DDL addresses it. */
+export interface DeclaredTable {
+  /** Present when the connection is `withSchema(...)`-bound. */
+  schema?: string
+  name: string
+}
+
+/** How a table reads in an error message: qualified only when it is. */
+const displayName = (table: DeclaredTable): string =>
+  table.schema ? `${table.schema}.${table.name}` : table.name
+
+/**
+ * `create table "workflow_runs"` and `create table "app"."workflow_runs"`.
+ *
+ * The schema half is optional and the table is always the LAST quoted
+ * identifier — matching the first one instead reads a `withSchema('app')`
+ * connection's DDL as a table called `app`, which is no table at all.
+ */
+const CREATE_TABLE =
+  /^\s*create table (?:if not exists\s+)?(?:"([^"]+)"\.)?"([^"]+)"/i
+
 /**
  * The tables a schema creates, read back out of its own compiled SQL.
  *
  * Derived rather than declared alongside the statements, because a hand-kept
  * list of names is a second source of truth: rename a table in a statement and
  * the list keeps answering for the old one.
+ *
+ * Compiled against `db` rather than in the abstract, so the answer carries the
+ * schema the DDL will actually target — the same binding the lookup has to use.
  */
-const declaredTables = (schema: PikkuSchema, db: Kysely<any>): string[] => {
+export const declaredTables = (
+  schema: PikkuSchema,
+  db: Kysely<any>
+): DeclaredTable[] => {
   const bound = bind(db)
-  const tables: string[] = []
+  const tables: DeclaredTable[] = []
   for (const statement of schema.statements) {
-    const match = /^\s*create table "([^"]+)"/i.exec(
-      statement(bound, {}).compile().sql
-    )
-    if (match) tables.push(match[1]!)
+    const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql)
+    if (match) tables.push({ schema: match[1], name: match[2]! })
   }
   return tables
 }
@@ -171,8 +250,51 @@ const declaredTables = (schema: PikkuSchema, db: Kysely<any>): string[] => {
 /** What `ensurePikkuSchema` found when it looked. */
 export type EnsureOutcome = 'present' | 'created'
 
-const tablesOf = async (db: Kysely<any>): Promise<Set<string>> =>
-  new Set((await bind(db).introspection.getTables()).map((table) => table.name))
+/**
+ * What `db` already has, indexed both ways.
+ *
+ * Introspection reports every schema it can see, so the bare name alone cannot
+ * answer the question: a `workflow_runs` in some other schema is not the one a
+ * `withSchema('app')` connection reads or writes. Qualified declarations are
+ * matched on the pair. Unqualified DDL resolves against the connection's
+ * search_path, which is not knowable from here, so it falls back to the name.
+ */
+interface ExistingTables {
+  qualified: Set<string>
+  bare: Set<string>
+  /** Whether this engine reports schemas at all. */
+  schemaAware: boolean
+}
+
+const tablesOf = async (db: Kysely<any>): Promise<ExistingTables> => {
+  const tables = await bind(db).introspection.getTables()
+  return {
+    qualified: new Set(
+      tables
+        .filter((table) => table.schema)
+        .map((table) => `${table.schema}.${table.name}`)
+    ),
+    bare: new Set(tables.map((table) => table.name)),
+    schemaAware: tables.some((table) => table.schema !== undefined),
+  }
+}
+
+/**
+ * Qualified declarations match on the pair — but only where that pair means
+ * something.
+ *
+ * `TableMetadata.schema` is optional, and sqlite and mysql leave it unset:
+ * sqlite has one schema to be in, and mysql calls the thing a database. A
+ * `withSchema('main')` connection there declares `main.workflow_runs` while
+ * introspection can only offer `workflow_runs`, and holding out for the pair
+ * would read every existing table as missing — the same false negative this
+ * whole function exists to fix, arrived at from the other side. So when the
+ * engine reports no schemas, the bare name is the only name there is.
+ */
+const isPresent = (existing: ExistingTables, table: DeclaredTable): boolean =>
+  table.schema && existing.schemaAware
+    ? existing.qualified.has(`${table.schema}.${table.name}`)
+    : existing.bare.has(table.name)
 
 /**
  * Make sure one schema's tables are there, creating them only if none are.
@@ -198,14 +320,17 @@ export const ensurePikkuSchema = async (
 ): Promise<EnsureOutcome> => {
   const existing = await tablesOf(db)
   const declared = declaredTables(schema, db)
-  const missing = declared.filter((table) => !existing.has(table))
+  const missing = declared.filter((table) => !isPresent(existing, table))
 
   if (missing.length === 0) return 'present'
 
   if (missing.length < declared.length) {
     throw new Error(
-      `The '${schema.name}' schema is half applied: ${missing.join(', ')} missing, ` +
-        `${declared.filter((table) => existing.has(table)).join(', ')} already there. ` +
+      `The '${schema.name}' schema is half applied: ${missing.map(displayName).join(', ')} missing, ` +
+        `${declared
+          .filter((table) => isPresent(existing, table))
+          .map(displayName)
+          .join(', ')} already there. ` +
         'Something else owns part of it. Reconcile it with a migration — ' +
         '`pikku db generate` writes the declaration down — rather than creating the rest at boot.'
     )
@@ -221,7 +346,7 @@ export const ensurePikkuSchema = async (
     // a guarantee — a boot race is only fully answered by having run the
     // migration, which is what `pikku db generate` is for.
     const after = await tablesOf(db)
-    if (declared.every((table) => after.has(table))) return 'present'
+    if (declared.every((table) => isPresent(after, table))) return 'present'
     throw error
   }
 }