@pikku/kysely 0.13.5 → 0.13.7

package/dist/src/kysely-workflow-service.js

@@ -1,4 +1,5 @@
 import { PikkuWorkflowService, } from '@pikku/core/workflow';
+import { sql } from 'kysely';
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js';
 import { parseJson } from './kysely-json.js';
 import { ensurePikkuSchema } from './schema/index.js';
@@ -68,11 +69,12 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             retries: stepOptions?.retries ?? null,
             retryDelay: stepOptions?.retryDelay?.toString() ?? null,
             fromStepName: fromStepName ?? null,
+            currentAttempt: 1,
             createdAt: now,
             updatedAt: now,
         })
             .execute();
-        await this.insertHistoryRecord(stepId, 'pending');
+        await this.insertHistoryRecord(stepId, 'pending', 1);
         return {
             stepId,
             status: 'pending',
@@ -88,25 +90,24 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     }
     async getStepState(runId, stepName) {
         const row = await this.db
-            .selectFrom('workflowStep as s')
+            .selectFrom('workflowStep')
             .select([
-            's.workflowStepId',
-            's.status',
-            's.result',
-            's.error',
-            's.retries',
-            's.retryDelay',
-            's.fromStepName',
-            's.createdAt',
-            's.updatedAt',
+            'workflowStepId',
+            'status',
+            'result',
+            'error',
+            'retries',
+            'retryDelay',
+            'fromStepName',
+            // `current_attempt` is bumped alongside every history insert, so it is
+            // already the count this used to aggregate — and reading it keeps the
+            // engine's hottest query off a table that grows for the life of the run.
+            'currentAttempt',
+            'createdAt',
+            'updatedAt',
         ])
-            .select((eb) => eb
-            .selectFrom('workflowStepHistory')
-            .select(eb.fn.countAll().as('cnt'))
-            .whereRef('workflowStepHistory.workflowStepId', '=', 's.workflowStepId')
-            .as('attemptCount'))
-            .where('s.workflowRunId', '=', runId)
-            .where('s.stepName', '=', stepName)
+            .where('workflowRunId', '=', runId)
+            .where('stepName', '=', stepName)
             .executeTakeFirst();
         if (!row) {
             throw new Error(`Step not found: runId=${runId}, stepName=${stepName}. Use insertStepState to create it.`);
@@ -116,7 +117,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             status: row.status,
             result: parseJson(row.result),
             error: parseJson(row.error),
-            attemptCount: Number(row.attemptCount),
+            attemptCount: Number(row.currentAttempt ?? 1),
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
             fromStepName: row.fromStepName ?? undefined,
@@ -124,29 +125,45 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             updatedAt: new Date(row.updatedAt),
         };
     }
+    async listStepStates(runId) {
+        const rows = await this.db
+            .selectFrom('workflowStep')
+            .select([
+            'workflowStepId',
+            'stepName',
+            'status',
+            'result',
+            'error',
+            'retries',
+            'retryDelay',
+            'fromStepName',
+            'currentAttempt',
+            'createdAt',
+            'updatedAt',
+        ])
+            .where('workflowRunId', '=', runId)
+            .execute();
+        return rows.map((row) => ({
+            stepId: row.workflowStepId,
+            stepName: row.stepName,
+            status: row.status,
+            result: parseJson(row.result),
+            error: parseJson(row.error),
+            attemptCount: Number(row.currentAttempt ?? 1),
+            retries: row.retries != null ? Number(row.retries) : undefined,
+            retryDelay: row.retryDelay ?? undefined,
+            fromStepName: row.fromStepName ?? undefined,
+            createdAt: new Date(row.createdAt),
+            updatedAt: new Date(row.updatedAt),
+        }));
+    }
     async getRunHistory(runId) {
         return this.runService.getRunHistory(runId);
     }
     async setStepRunningImpl(stepId) {
-        await this.db
-            .updateTable('workflowStep')
-            .set({ status: 'running', updatedAt: new Date() })
-            .where('workflowStepId', '=', stepId)
-            .execute();
-        const latestHistory = await this.db
-            .selectFrom('workflowStepHistory')
-            .select('historyId')
-            .where('workflowStepId', '=', stepId)
-            .orderBy('createdAt', 'desc')
-            .limit(1)
-            .executeTakeFirst();
-        if (latestHistory) {
-            await this.db
-                .updateTable('workflowStepHistory')
-                .set({ status: 'running', runningAt: new Date() })
-                .where('historyId', '=', latestHistory.historyId)
-                .execute();
-        }
+        await this.writeStepTransition(stepId, 'running', {
+            runningAt: new Date(),
+        });
     }
     async setStepScheduledImpl(stepId) {
         await this.db
@@ -155,12 +172,85 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             .where('workflowStepId', '=', stepId)
             .execute();
     }
-    async insertHistoryRecord(stepId, status, result, error) {
+    /**
+     * Move a step and its current history attempt to the same status in one
+     * transaction.
+     *
+     * Both halves matter: a crash between them used to leave the step row saying
+     * `succeeded` while its history row still said `running`, which silently
+     * corrupts `getRunHistory` and every timeline reconstruction built on it.
+     * The history row is targeted by its `attempt` rather than by a `LIMIT 1`
+     * over `created_at`, so a retry that lands in the same millisecond as the
+     * attempt it replaces still resolves the newer row.
+     */
+    async writeStepTransition(stepId, status, historyValues) {
+        const now = new Date();
+        const stepValues = { status, updatedAt: now };
+        if ('result' in historyValues)
+            stepValues.result = historyValues.result;
+        if ('error' in historyValues)
+            stepValues.error = historyValues.error;
+        // A step carries only its latest outcome, so the other half is cleared.
+        if (status === 'succeeded')
+            stepValues.error = null;
+        if (status === 'failed')
+            stepValues.result = null;
+        await this.db.transaction().execute(async (trx) => {
+            await trx
+                .updateTable('workflowStep')
+                .set({
+                ...stepValues,
+                // A step that somehow reaches a transition without a live attempt
+                // addresses no history row at all, so give it one here. Written as an
+                // expression over the row's own column rather than a subquery, which
+                // MySQL would reject against the table being updated.
+                currentAttempt: sql `coalesce(current_attempt, 1)`,
+            })
+                .where('workflowStepId', '=', stepId)
+                .execute();
+            const written = await trx
+                .updateTable('workflowStepHistory')
+                .set({ status, ...historyValues })
+                .where('workflowStepId', '=', stepId)
+                // Correlate through the step row rather than a MAX() over this table:
+                // MySQL rejects a subquery that reads the table being updated, and a
+                // primary-key lookup is cheaper than an aggregate anyway.
+                .where('attempt', '=', (eb) => eb
+                .selectFrom('workflowStep')
+                .select('currentAttempt')
+                .where('workflowStepId', '=', stepId))
+                .executeTakeFirst();
+            // Matching nothing means the attempt has no history row to move, which
+            // would commit the step's new status with no record of it — precisely
+            // the divergence this transaction exists to prevent. Write the missing
+            // row instead of letting the pair drift apart.
+            if (Number(written?.numUpdatedRows ?? 0n) === 0) {
+                const step = await trx
+                    .selectFrom('workflowStep')
+                    .select('currentAttempt')
+                    .where('workflowStepId', '=', stepId)
+                    .executeTakeFirst();
+                await trx
+                    .insertInto('workflowStepHistory')
+                    .values({
+                    historyId: crypto.randomUUID(),
+                    workflowStepId: stepId,
+                    status,
+                    attempt: step?.currentAttempt ?? 1,
+                    createdAt: now,
+                    ...historyValues,
+                })
+                    .execute();
+            }
+        });
+    }
+    async insertHistoryRecord(stepId, status, attempt, result, error) {
         const now = new Date();
         const values = {
             historyId: crypto.randomUUID(),
             workflowStepId: stepId,
             status,
+            attempt,
             result: result != null ? JSON.stringify(result) : null,
             error: error != null ? JSON.stringify(error) : null,
             createdAt: now,
@@ -199,35 +289,10 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             .execute();
     }
     async setStepResultImpl(stepId, result) {
-        const resultJson = JSON.stringify(result);
-        await this.db
-            .updateTable('workflowStep')
-            .set({
-            status: 'succeeded',
-            result: resultJson,
-            error: null,
-            updatedAt: new Date(),
-        })
-            .where('workflowStepId', '=', stepId)
-            .execute();
-        const latestHistory = await this.db
-            .selectFrom('workflowStepHistory')
-            .select('historyId')
-            .where('workflowStepId', '=', stepId)
-            .orderBy('createdAt', 'desc')
-            .limit(1)
-            .executeTakeFirst();
-        if (latestHistory) {
-            await this.db
-                .updateTable('workflowStepHistory')
-                .set({
-                status: 'succeeded',
-                result: resultJson,
-                succeededAt: new Date(),
-            })
-                .where('historyId', '=', latestHistory.historyId)
-                .execute();
-        }
+        await this.writeStepTransition(stepId, 'succeeded', {
+            result: JSON.stringify(result),
+            succeededAt: new Date(),
+        });
     }
     async setStepErrorImpl(stepId, error) {
         const serializedError = {
@@ -235,65 +300,54 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             stack: error.stack,
             code: error.code,
         };
-        const errorJson = JSON.stringify(serializedError);
+        await this.writeStepTransition(stepId, 'failed', {
+            error: JSON.stringify(serializedError),
+            failedAt: new Date(),
+        });
+    }
+    async createRetryAttemptImpl(stepId, status) {
+        // Read before writing: the new attempt number comes from the history this
+        // step already has, and the row it reads is the one returned below.
+        const previous = await this.db
+            .selectFrom('workflowStep')
+            .select('currentAttempt')
+            .where('workflowStepId', '=', stepId)
+            .executeTakeFirst();
+        const attempt = (previous?.currentAttempt ?? 0) + 1;
         await this.db
             .updateTable('workflowStep')
             .set({
-            status: 'failed',
-            error: errorJson,
+            status,
             result: null,
+            error: null,
+            currentAttempt: attempt,
             updatedAt: new Date(),
         })
             .where('workflowStepId', '=', stepId)
             .execute();
-        const latestHistory = await this.db
-            .selectFrom('workflowStepHistory')
-            .select('historyId')
-            .where('workflowStepId', '=', stepId)
-            .orderBy('createdAt', 'desc')
-            .limit(1)
-            .executeTakeFirst();
-        if (latestHistory) {
-            await this.db
-                .updateTable('workflowStepHistory')
-                .set({ status: 'failed', error: errorJson, failedAt: new Date() })
-                .where('historyId', '=', latestHistory.historyId)
-                .execute();
-        }
-    }
-    async createRetryAttemptImpl(stepId, status) {
-        await this.db
-            .updateTable('workflowStep')
-            .set({ status, result: null, error: null, updatedAt: new Date() })
-            .where('workflowStepId', '=', stepId)
-            .execute();
-        await this.insertHistoryRecord(stepId, status);
+        await this.insertHistoryRecord(stepId, status, attempt);
         const row = await this.db
-            .selectFrom('workflowStep as s')
+            .selectFrom('workflowStep')
             .select([
-            's.workflowStepId',
-            's.status',
-            's.result',
-            's.error',
-            's.retries',
-            's.retryDelay',
-            's.fromStepName',
-            's.createdAt',
-            's.updatedAt',
+            'workflowStepId',
+            'status',
+            'result',
+            'error',
+            'retries',
+            'retryDelay',
+            'fromStepName',
+            'currentAttempt',
+            'createdAt',
+            'updatedAt',
         ])
-            .select((eb) => eb
-            .selectFrom('workflowStepHistory')
-            .select(eb.fn.countAll().as('cnt'))
-            .whereRef('workflowStepHistory.workflowStepId', '=', 's.workflowStepId')
-            .as('attemptCount'))
-            .where('s.workflowStepId', '=', stepId)
+            .where('workflowStepId', '=', stepId)
             .executeTakeFirstOrThrow();
         return {
             stepId: row.workflowStepId,
             status: row.status,
             result: parseJson(row.result),
             error: parseJson(row.error),
-            attemptCount: Number(row.attemptCount),
+            attemptCount: Number(row.currentAttempt ?? 1),
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
             fromStepName: row.fromStepName ?? undefined,
@@ -339,18 +393,6 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         }
         return { completedNodeIds, failedNodeIds, branchKeys };
     }
-    async getNodesWithoutSteps(runId, nodeIds) {
-        if (nodeIds.length === 0)
-            return [];
-        const result = await this.db
-            .selectFrom('workflowStep')
-            .select('stepName')
-            .where('workflowRunId', '=', runId)
-            .where('stepName', 'in', nodeIds)
-            .execute();
-        const existingStepNames = new Set(result.map((r) => r.stepName));
-        return nodeIds.filter((id) => !existingStepNames.has(id));
-    }
     async getStepInstances(runId) {
         const rows = await this.db
             .selectFrom('workflowStep')
@@ -386,17 +428,39 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             .where('workflowStepId', '=', stepId)
             .execute();
     }
+    /**
+     * Merge one key into the run's JSON state, as a single expression evaluated
+     * by the database.
+     *
+     * The SQL is dialect-specific, so subclasses override this rather than the
+     * caller. The default is the SQLite form; MySQL and Postgres differ in how
+     * they cast a JSON literal.
+     *
+     * @param path - JSON path to the key, already quoted (`$."key"`)
+     * @param json - The value as JSON text
+     */
+    jsonSetState(path, json) {
+        return sql `json_set(coalesce(state, '{}'), ${path}, json(${json}))`;
+    }
+    /**
+     * A JSON path for an arbitrary key. State keys are user-supplied (a graph's
+     * `setState` name, an approval's hex-encoded reason), so the key is quoted
+     * rather than interpolated bare — an unquoted `$.a.b` would silently address
+     * a nested path instead of the key literally called `a.b`.
+     */
+    jsonPathFor(name) {
+        return `$.${JSON.stringify(name)}`;
+    }
     async updateRunStateImpl(runId, name, value) {
-        const row = await this.db
-            .selectFrom('workflowRuns')
-            .select('state')
-            .where('workflowRunId', '=', runId)
-            .executeTakeFirst();
-        const state = parseJson(row?.state) ?? {};
-        state[name] = value;
+        // Merged by the database, not read-modify-written in JS: two graph nodes
+        // calling setState concurrently used to race, and the later write silently
+        // dropped whichever key the earlier one had added.
         await this.db
             .updateTable('workflowRuns')
-            .set({ state: JSON.stringify(state), updatedAt: new Date() })
+            .set({
+            state: this.jsonSetState(this.jsonPathFor(name), JSON.stringify(value ?? null)),
+            updatedAt: new Date(),
+        })
             .where('workflowRunId', '=', runId)
             .execute();
     }
@@ -434,21 +498,5 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     async getWorkflowVersion(name, graphHash) {
         return this.runService.getWorkflowVersion(name, graphHash);
     }
-    async getAIGeneratedWorkflows(agentName) {
-        let query = this.db
-            .selectFrom('workflowVersions')
-            .select(['workflowName', 'graphHash', 'graph'])
-            .where('source', '=', 'dynamic-workflow')
-            .where('status', '=', 'active');
-        if (agentName) {
-            query = query.where('workflowName', 'like', `ai:${agentName}:%`);
-        }
-        const rows = await query.execute();
-        return rows.map((row) => ({
-            workflowName: row.workflowName,
-            graphHash: row.graphHash,
-            graph: typeof row.graph === 'string' ? JSON.parse(row.graph) : row.graph,
-        }));
-    }
     async close() { }
 }

package/dist/src/schema/credential.schema.js

@@ -22,6 +22,11 @@ export const credentialSchema = {
             .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
         rawStatement(sql `CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`),
+        (db) => db.schema
+            .createTable('credentialKekSalts')
+            .addColumn('keyVersion', 'integer', (col) => col.primaryKey())
+            .addColumn('salt', 'varchar(64)', (col) => col.notNull())
+            .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
         (db) => db.schema
             .createTable('credentialsAudit')
             .addColumn('id', 'varchar(36)', (col) => col.primaryKey())

package/dist/src/schema/scope.schema.js

@@ -29,6 +29,18 @@ export const scopeSchema = {
             .createTable('pikkuRoles')
             .addColumn('name', 'text', (col) => col.primaryKey())
             .addColumn('description', 'text')
+            // Declared in code with `defineSystemRole` rather than composed by an
+            // admin. What it buys is the refusals: a system role cannot be renamed,
+            // re-scoped or deleted from the console, and a console role cannot be
+            // created with its name — two rows answering to one name would make
+            // "does Susan hold `buyer`?" depend on which one the store returned.
+            .addColumn('system', 'boolean', (col) => col.defaultTo(false).notNull())
+            // A system role whose declaration has gone: still held by whoever holds
+            // it, no longer offered for new grants, awaiting `pikku roles prune`.
+            // The same additive contract as `pikkuScopes.declared`, and for the same
+            // reason — a mid-deploy revocation is not something a code edit should
+            // be able to cause.
+            .addColumn('declared', 'boolean', (col) => col.defaultTo(true).notNull())
             .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
         (db) => db.schema
             .createTable('pikkuRoleScopes')

package/dist/src/schema/secret.schema.js

@@ -19,6 +19,11 @@ export const secretSchema = {
             .addColumn('keyVersion', 'integer', (col) => col.notNull())
             .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
+        (db) => db.schema
+            .createTable('secretKekSalts')
+            .addColumn('keyVersion', 'integer', (col) => col.primaryKey())
+            .addColumn('salt', 'varchar(64)', (col) => col.notNull())
+            .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
         (db) => db.schema
             .createTable('secretsAudit')
             .addColumn('id', 'varchar(36)', (col) => col.primaryKey())

package/dist/src/schema/workflow.schema.d.ts

@@ -6,8 +6,10 @@ import type { PikkuSchema } from './pikku-schema.types.js';
  * primary keys had `defaultTo(sql.raw("'" + crypto.randomUUID() + "'"))`, which
  * evaluates once while the statement is built — every row would have taken the
  * same default, so it was never a generator; the services supply the id. And
- * `workflowStep.fromStepName` was backfilled by an `alterTable(...).catch(() =>
- * {})` bolted on after the fact, which is the job a migration does.
+ * `workflowStep.fromStepName`, `workflowStep.currentAttempt` and
+ * `workflowStepHistory.attempt` were each backfilled by an
+ * `alterTable(...).catch(() => {})` bolted on after the fact, which is the job a
+ * migration does — they are declared here and nowhere else.
  *
  * `KyselyWorkflowMirror` created these same four tables separately. There is one
  * declaration now, and both services read it.

package/dist/src/schema/workflow.schema.js

@@ -6,8 +6,10 @@ import { sql } from 'kysely';
  * primary keys had `defaultTo(sql.raw("'" + crypto.randomUUID() + "'"))`, which
  * evaluates once while the statement is built — every row would have taken the
  * same default, so it was never a generator; the services supply the id. And
- * `workflowStep.fromStepName` was backfilled by an `alterTable(...).catch(() =>
- * {})` bolted on after the fact, which is the job a migration does.
+ * `workflowStep.fromStepName`, `workflowStep.currentAttempt` and
+ * `workflowStepHistory.attempt` were each backfilled by an
+ * `alterTable(...).catch(() => {})` bolted on after the fact, which is the job a
+ * migration does — they are declared here and nowhere else.
  *
  * `KyselyWorkflowMirror` created these same four tables separately. There is one
  * declaration now, and both services read it.
@@ -49,6 +51,7 @@ export const workflowSchema = {
             .addColumn('retries', 'integer')
             .addColumn('retryDelay', 'text')
             .addColumn('fromStepName', 'text')
+            .addColumn('currentAttempt', 'integer')
             .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addUniqueConstraint('workflow_step_run_name_unique', [
@@ -65,6 +68,7 @@ export const workflowSchema = {
             .addColumn('status', 'text', (col) => col.notNull())
             .addColumn('result', 'text')
             .addColumn('error', 'text')
+            .addColumn('attempt', 'integer')
             .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addColumn('runningAt', 'timestamp')
             .addColumn('scheduledAt', 'timestamp')
@@ -82,5 +86,27 @@ export const workflowSchema = {
             'workflowName',
             'graphHash',
         ]),
+        // The indexes trail the tables so that a table is never indexed before it
+        // exists, whatever order the declarations are read in.
+        (db) => db.schema
+            .createIndex('idx_workflow_runs_status_created')
+            .on('workflowRuns')
+            .columns(['status', 'createdAt']),
+        (db) => db.schema
+            .createIndex('idx_workflow_runs_workflow_created')
+            .on('workflowRuns')
+            .columns(['workflow', 'createdAt']),
+        (db) => db.schema
+            .createIndex('idx_workflow_step_run_status')
+            .on('workflowStep')
+            .columns(['workflowRunId', 'status']),
+        (db) => db.schema
+            .createIndex('idx_workflow_step_history_step')
+            .on('workflowStepHistory')
+            .columns(['workflowStepId', 'createdAt']),
+        (db) => db.schema
+            .createIndex('idx_workflow_versions_source_status')
+            .on('workflowVersions')
+            .columns(['source', 'status']),
     ],
 };

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@pikku/kysely",
-  "version": "0.13.5",
+  "version": "0.13.7",
+  "description": "Kysely-backed Pikku services — workflow state, secrets, sessions, scopes and audit",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/src/index.js",
@@ -20,7 +21,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.69"
+    "@pikku/core": "^0.12.74"
   },
   "dependencies": {
     "kysely": "^0.29.0"