@pikku/kysely 0.13.5 → 0.13.7

package/dist/src/kysely-scope-service.js

@@ -1,3 +1,4 @@
+import { SystemRoleImmutableError, SystemRoleShadowedError, } from '@pikku/core/errors';
 import { ensurePikkuSchema } from './schema/index.js';
 import { scopeSchema } from './schema/scope.schema.js';
 /**
@@ -79,11 +80,69 @@ export class KyselyScopeService {
             declared: !!row.declared,
         }));
     }
+    /**
+     * Registers the roles declared with `defineSystemRole`.
+     *
+     * Additive on the same terms as `syncScopes` — a role whose declaration has
+     * gone is marked `declared = false` rather than deleted, so a rollback or a
+     * rolling deploy cannot strip everyone's grant. Its *scope set* is replaced
+     * outright, because that is the declaration's entire content: editing
+     * `defineSystemRole` is how you change what a role means, and the deploy is
+     * when it takes effect.
+     */
+    async syncSystemRoles(roles) {
+        await this.db.transaction().execute(async (trx) => {
+            for (const role of roles) {
+                await trx
+                    .insertInto('pikkuRoles')
+                    .values({
+                    name: role.name,
+                    description: role.description ?? null,
+                    system: true,
+                    declared: true,
+                })
+                    .onConflict((oc) => oc.column('name').doUpdateSet((eb) => ({
+                    description: eb.ref('excluded.description'),
+                    system: true,
+                    declared: true,
+                })))
+                    .execute();
+                await trx
+                    .deleteFrom('pikkuRoleScopes')
+                    .where('role', '=', role.name)
+                    .execute();
+                if (role.scopes.length > 0) {
+                    await trx
+                        .insertInto('pikkuRoleScopes')
+                        .values(role.scopes.map((scope) => ({ role: role.name, scope })))
+                        .execute();
+                }
+            }
+            // Only system rows are in scope here: an admin's own role is not
+            // undeclared, it was never declared, and marking it would take it out of
+            // the console for a reason nobody could act on.
+            const markStale = trx
+                .updateTable('pikkuRoles')
+                .set({ declared: false })
+                .where('system', '=', true);
+            await (roles.length > 0
+                ? markStale.where('name', 'not in', roles.map((role) => role.name))
+                : markStale).execute();
+        });
+    }
     async createRole(role) {
+        if (await this.isSystemRole(role.name)) {
+            throw new SystemRoleShadowedError(role.name);
+        }
         await this.db.transaction().execute(async (trx) => {
             await trx
                 .insertInto('pikkuRoles')
-                .values({ name: role.name, description: role.description ?? null })
+                .values({
+                name: role.name,
+                description: role.description ?? null,
+                system: false,
+                declared: true,
+            })
                 .execute();
             if (role.scopes.length > 0) {
                 await trx
@@ -94,9 +153,15 @@ export class KyselyScopeService {
         });
     }
     async deleteRole(name) {
+        if (await this.isSystemRole(name)) {
+            throw new SystemRoleImmutableError(name, 'delete');
+        }
         await this.db.deleteFrom('pikkuRoles').where('name', '=', name).execute();
     }
     async setRoleScopes(name, scopes) {
+        if (await this.isSystemRole(name)) {
+            throw new SystemRoleImmutableError(name, 're-scope');
+        }
         await this.db.transaction().execute(async (trx) => {
             await trx.deleteFrom('pikkuRoleScopes').where('role', '=', name).execute();
             if (scopes.length > 0) {
@@ -107,10 +172,25 @@ export class KyselyScopeService {
             }
         });
     }
+    /**
+     * Whether the store holds this name as a system role.
+     *
+     * Asked of the store rather than of the generated `SYSTEM_ROLES`, because the
+     * store is where a shadow would actually collide — and because a role whose
+     * declaration was deleted is still immutable until someone prunes it.
+     */
+    async isSystemRole(name) {
+        const row = await this.db
+            .selectFrom('pikkuRoles')
+            .select('system')
+            .where('name', '=', name)
+            .executeTakeFirst();
+        return !!row?.system;
+    }
     async listRoles() {
         const roles = await this.db
             .selectFrom('pikkuRoles')
-            .select(['name', 'description'])
+            .select(['name', 'description', 'system', 'declared'])
             .execute();
         const scopeRows = await this.db
             .selectFrom('pikkuRoleScopes')
@@ -130,6 +210,8 @@ export class KyselyScopeService {
             name: role.name,
             description: role.description ?? undefined,
             scopes: byRole.get(role.name) ?? [],
+            system: !!role.system,
+            declared: !!role.declared,
         }));
     }
     async addUserToRole(userId, role, grantedBy) {
@@ -215,4 +297,45 @@ export class KyselyScopeService {
             .execute();
         return deleted.map((r) => r.name);
     }
+    /**
+     * System roles the last sync found no declaration for, with how many people
+     * still hold each. Powers `pikku roles audit`.
+     *
+     * The count is the whole point: "this role is gone from the code" and "this
+     * role is gone from the code and 40 people are standing on it" call for
+     * different decisions, and the number is what makes pruning a choice rather
+     * than a formality.
+     */
+    async findStaleSystemRoles() {
+        const stale = await this.db
+            .selectFrom('pikkuRoles')
+            .leftJoin('pikkuUserRole', 'pikkuUserRole.role', 'pikkuRoles.name')
+            .select(['pikkuRoles.name as role', 'pikkuUserRole.userId as userId'])
+            .where('pikkuRoles.system', '=', true)
+            .where('pikkuRoles.declared', '=', false)
+            .execute();
+        const byRole = new Map();
+        for (const row of stale) {
+            byRole.set(row.role, (byRole.get(row.role) ?? 0) + (row.userId ? 1 : 0));
+        }
+        return [...byRole.entries()].map(([role, users]) => ({ role, users }));
+    }
+    /**
+     * Removes undeclared system roles, cascading them out of every user grant
+     * that holds them. This revokes access, so it is never run implicitly.
+     */
+    async pruneSystemRoles() {
+        const stale = await this.findStaleSystemRoles();
+        if (stale.length === 0) {
+            return [];
+        }
+        const deleted = await this.db
+            .deleteFrom('pikkuRoles')
+            .where('name', 'in', stale.map((s) => s.role))
+            .where('system', '=', true)
+            .where('declared', '=', false)
+            .returning('name')
+            .execute();
+        return deleted.map((r) => r.name);
+    }
 }

package/dist/src/kysely-secret-service.d.ts

@@ -11,6 +11,8 @@ export interface KyselySecretServiceConfig {
 export declare class KyselySecretService implements SecretService {
     private db;
     private initialized;
+    private kekSalts;
+    private keks;
     private key;
     private keyVersion;
     private previousKey?;
@@ -19,6 +21,7 @@ export declare class KyselySecretService implements SecretService {
     constructor(db: Kysely<KyselyPikkuDB>, config: KyselySecretServiceConfig);
     init(): Promise<void>;
     private logAudit;
+    private getKEKSalt;
     private getKEK;
     getSecret<T = string>(key: string): Promise<T>;
     hasSecret(key: string): Promise<boolean>;

package/dist/src/kysely-secret-service.js

@@ -1,9 +1,11 @@
-import { envelopeEncrypt, envelopeDecrypt, envelopeRewrap, } from '@pikku/core/crypto-utils';
+import { deriveKEK, generateKEKSalt, envelopeEncrypt, envelopeDecrypt, envelopeRewrap, } from '@pikku/core/crypto-utils';
 import { ensurePikkuSchema } from './schema/index.js';
 import { secretSchema } from './schema/secret.schema.js';
 export class KyselySecretService {
     db;
     initialized = false;
+    kekSalts = new Map();
+    keks = new Map();
     key;
     keyVersion;
     previousKey;
@@ -38,12 +40,47 @@ export class KyselySecretService {
         })
             .execute();
     }
-    getKEK(version) {
-        if (version === this.keyVersion)
-            return this.key;
-        if (this.previousKey)
-            return this.previousKey;
-        throw new Error(`No KEK available for key_version ${version}`);
+    async getKEKSalt(version) {
+        const cached = this.kekSalts.get(version);
+        if (cached)
+            return cached;
+        const existing = await this.db
+            .selectFrom('secretKekSalts')
+            .select('salt')
+            .where('keyVersion', '=', version)
+            .executeTakeFirst();
+        let salt = existing?.salt;
+        if (!salt) {
+            await this.db
+                .insertInto('secretKekSalts')
+                .values({
+                keyVersion: version,
+                salt: generateKEKSalt(),
+                createdAt: new Date().toISOString(),
+            })
+                .onConflict((oc) => oc.column('keyVersion').doNothing())
+                .execute();
+            const row = await this.db
+                .selectFrom('secretKekSalts')
+                .select('salt')
+                .where('keyVersion', '=', version)
+                .executeTakeFirstOrThrow();
+            salt = row.salt;
+        }
+        this.kekSalts.set(version, salt);
+        return salt;
+    }
+    async getKEK(version) {
+        const cached = this.keks.get(version);
+        if (cached)
+            return cached;
+        const passphrase = version === this.keyVersion ? this.key : this.previousKey;
+        if (!passphrase) {
+            throw new Error(`No KEK available for key_version ${version}`);
+        }
+        const kek = await deriveKEK(passphrase, await this.getKEKSalt(version));
+        this.keks.set(version, kek);
+        return kek;
     }
     async getSecret(key) {
         const row = await this.db
@@ -53,7 +90,7 @@ export class KyselySecretService {
             .executeTakeFirst();
         if (!row)
             throw new Error('Requested secret not found');
-        const kek = this.getKEK(row.keyVersion);
+        const kek = await this.getKEK(row.keyVersion);
         const result = await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek);
         await this.logAudit(key, 'read');
         return result;
@@ -67,7 +104,7 @@ export class KyselySecretService {
         return !!row;
     }
     async setSecret(key, value) {
-        const { ciphertext, wrappedDEK } = await envelopeEncrypt(this.key, value);
+        const { ciphertext, wrappedDEK } = await envelopeEncrypt(await this.getKEK(this.keyVersion), value);
         const now = new Date().toISOString();
         await this.db
             .insertInto('secrets')
@@ -101,7 +138,7 @@ export class KyselySecretService {
         const out = {};
         for (const row of rows) {
             try {
-                const kek = this.getKEK(row.keyVersion);
+                const kek = await this.getKEK(row.keyVersion);
                 out[row.key] = await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek);
             }
             catch (cause) {
@@ -126,8 +163,10 @@ export class KyselySecretService {
             .select(['key', 'wrappedDek'])
             .where('keyVersion', '<', this.keyVersion)
             .execute();
+        const oldKEK = await this.getKEK(this.keyVersion - 1);
+        const newKEK = await this.getKEK(this.keyVersion);
         for (const row of rows) {
-            const newWrappedDEK = await envelopeRewrap(this.previousKey, this.key, row.wrappedDek);
+            const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek);
             await this.db
                 .updateTable('secrets')
                 .set({

package/dist/src/kysely-tables.d.ts

@@ -44,6 +44,8 @@ export interface WorkflowStepTable {
     retries: number | null;
     retryDelay: string | null;
     fromStepName: string | null;
+    /** The history attempt currently in flight; see WorkflowStepHistoryTable.attempt. */
+    currentAttempt: number | null;
     createdAt: Generated<Date>;
     updatedAt: Generated<Date>;
 }
@@ -53,6 +55,12 @@ export interface WorkflowStepHistoryTable {
     status: StepStatus;
     result: string | null;
     error: string | null;
+    /**
+     * Monotonic attempt number within the step, starting at 1. Orders attempts
+     * and identifies the live one without relying on `createdAt`, which cannot
+     * separate a retry from the attempt it replaces in the same millisecond.
+     */
+    attempt: number | null;
     createdAt: Generated<Date>;
     runningAt: Date | null;
     scheduledAt: Date | null;
@@ -108,7 +116,12 @@ export interface AIRunTable {
     agentName: string;
     threadId: string;
     resourceId: string;
-    status: Generated<'running' | 'suspended' | 'completed' | 'failed'>;
+    /**
+     * `interrupted` is distinct from `failed`: the run was stopped on purpose,
+     * usually because the user talked over it. Nothing went wrong, so it should
+     * not show up in error reporting.
+     */
+    status: Generated<'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'>;
     errorMessage: string | null;
     suspendReason: 'approval' | 'credential' | 'rpc-missing' | null;
     missingRpcs: string | null;
@@ -138,6 +151,11 @@ export interface SecretsTable {
     createdAt: Generated<Date>;
     updatedAt: Generated<Date>;
 }
+export interface KekSaltsTable {
+    keyVersion: number;
+    salt: string;
+    createdAt: Generated<Date>;
+}
 export interface SecretsAuditTable {
     id: string;
     secretKey: string;
@@ -183,10 +201,14 @@ export interface PikkuScopesTable {
      */
     declared: Generated<boolean>;
 }
-/** A role: an admin-composed bag of scopes. */
+/** A role: a bag of scopes, composed by an admin or declared in code. */
 export interface PikkuRolesTable {
     name: string;
     description: string | null;
+    /** Declared with `defineSystemRole`, and therefore not editable here. */
+    system: Generated<boolean>;
+    /** False for a system role whose declaration has gone: held, but inert. */
+    declared: Generated<boolean>;
     createdAt: Generated<Date>;
 }
 export interface PikkuRoleScopesTable {
@@ -246,8 +268,10 @@ export interface KyselyPikkuDB {
     pikkuDeployments: PikkuDeploymentsTable;
     pikkuDeploymentFunctions: PikkuDeploymentFunctionsTable;
     secrets: SecretsTable;
+    secretKekSalts: KekSaltsTable;
     secretsAudit: SecretsAuditTable;
     credentials: CredentialsTable;
+    credentialKekSalts: KekSaltsTable;
     credentialsAudit: CredentialsAuditTable;
     pikkuUserSessions: UserSessionsTable;
     webhookDelivery: WebhookDeliveryTable;

package/dist/src/kysely-workflow-mirror.js

@@ -77,11 +77,12 @@ export class KyselyWorkflowMirror {
             retries: step.retries ?? null,
             retryDelay: step.retryDelay?.toString() ?? null,
             fromStepName: step.fromStepName ?? null,
+            currentAttempt: 1,
             createdAt: now,
             updatedAt: now,
         })
             .execute();
-        await this.insertHistoryRecord(step.stepId, step.status);
+        await this.insertHistoryRecord(step.stepId, step.status, 1);
     }
     async setStepRunning(stepId) {
         await this.updateStepStatus(stepId, 'running');
@@ -126,17 +127,24 @@ export class KyselyWorkflowMirror {
         await this.appendOrUpdateLatestHistory(stepId, 'failed', undefined, errorJson);
     }
     async createRetryAttempt(failedStepId, newStep) {
+        const previous = await this.db
+            .selectFrom('workflowStep')
+            .select('currentAttempt')
+            .where('workflowStepId', '=', failedStepId)
+            .executeTakeFirst();
+        const attempt = (previous?.currentAttempt ?? 0) + 1;
         await this.db
             .updateTable('workflowStep')
             .set({
             status: newStep.status,
             result: null,
             error: null,
+            currentAttempt: attempt,
             updatedAt: new Date(),
         })
             .where('workflowStepId', '=', failedStepId)
             .execute();
-        await this.insertHistoryRecord(failedStepId, newStep.status);
+        await this.insertHistoryRecord(failedStepId, newStep.status, attempt);
     }
     async setBranchTaken(stepId, branchKey) {
         await this.db
@@ -192,12 +200,13 @@ export class KyselyWorkflowMirror {
             .where('workflowStepId', '=', stepId)
             .execute();
     }
-    async insertHistoryRecord(stepId, status, resultJson, errorJson) {
+    async insertHistoryRecord(stepId, status, attempt, resultJson, errorJson) {
         const now = new Date();
         const values = {
             historyId: crypto.randomUUID(),
             workflowStepId: stepId,
             status,
+            attempt,
             result: resultJson ?? null,
             error: errorJson ?? null,
             createdAt: now,
@@ -219,30 +228,33 @@ export class KyselyWorkflowMirror {
      * returns the same sequence the executor would have produced.
      */
     async appendOrUpdateLatestHistory(stepId, status, resultJson, errorJson) {
-        const latest = await this.db
-            .selectFrom('workflowStepHistory')
-            .select('historyId')
+        const update = { status };
+        if (resultJson !== undefined)
+            update.result = resultJson;
+        if (errorJson !== undefined)
+            update.error = errorJson;
+        const tsField = timestampFieldFor(status);
+        if (tsField)
+            update[tsField] = new Date();
+        const result = await this.db
+            .updateTable('workflowStepHistory')
+            .set(update)
             .where('workflowStepId', '=', stepId)
-            .orderBy('createdAt', 'desc')
-            .limit(1)
+            // Correlate through the step row rather than sorting this table by
+            // createdAt: two attempts can land in the same millisecond, and the
+            // step row already knows which attempt is current.
+            .where('attempt', '=', (eb) => eb
+            .selectFrom('workflowStep')
+            .select('currentAttempt')
+            .where('workflowStepId', '=', stepId))
             .executeTakeFirst();
-        if (latest) {
-            const update = { status };
-            if (resultJson !== undefined)
-                update.result = resultJson;
-            if (errorJson !== undefined)
-                update.error = errorJson;
-            const tsField = timestampFieldFor(status);
-            if (tsField)
-                update[tsField] = new Date();
-            await this.db
-                .updateTable('workflowStepHistory')
-                .set(update)
-                .where('historyId', '=', latest.historyId)
-                .execute();
-        }
-        else {
-            await this.insertHistoryRecord(stepId, status, resultJson, errorJson);
+        if (Number(result?.numUpdatedRows ?? 0n) === 0) {
+            const step = await this.db
+                .selectFrom('workflowStep')
+                .select('currentAttempt')
+                .where('workflowStepId', '=', stepId)
+                .executeTakeFirst();
+            await this.insertHistoryRecord(stepId, status, step?.currentAttempt ?? 1, resultJson, errorJson);
         }
     }
 }

package/dist/src/kysely-workflow-run-service.d.ts

@@ -24,11 +24,6 @@ export declare class KyselyWorkflowRunService implements WorkflowRunService {
         graph: any;
         source: string;
     } | null>;
-    getAIGeneratedWorkflows(agentName?: string): Promise<Array<{
-        workflowName: string;
-        graphHash: string;
-        graph: any;
-    }>>;
     deleteRun(id: string): Promise<boolean>;
     private mapRunRow;
 }

package/dist/src/kysely-workflow-run-service.js

@@ -143,6 +143,7 @@ export class KyselyWorkflowRunService {
             'h.status',
             'h.result',
             'h.error',
+            'h.attempt',
             'h.createdAt',
             'h.runningAt',
             'h.scheduledAt',
@@ -150,7 +151,10 @@ export class KyselyWorkflowRunService {
             'h.failedAt',
         ])
             .where('s.workflowRunId', '=', runId)
+            // `attempt` breaks the tie when a retry lands in the same millisecond as
+            // the attempt it replaces, which `createdAt` alone cannot order.
             .orderBy('h.createdAt', 'asc')
+            .orderBy('h.attempt', 'asc')
             .execute();
         let attemptCounters = {};
         return result.map((row) => {
@@ -206,22 +210,6 @@ export class KyselyWorkflowRunService {
             source: row.source,
         };
     }
-    async getAIGeneratedWorkflows(agentName) {
-        let query = this.db
-            .selectFrom('workflowVersions')
-            .select(['workflowName', 'graphHash', 'graph'])
-            .where('source', '=', 'dynamic-workflow')
-            .where('status', '=', 'active');
-        if (agentName) {
-            query = query.where('workflowName', 'like', `ai:${agentName}:%`);
-        }
-        const rows = await query.execute();
-        return rows.map((row) => ({
-            workflowName: row.workflowName,
-            graphHash: row.graphHash,
-            graph: parseJson(row.graph),
-        }));
-    }
     async deleteRun(id) {
         const result = await this.db
             .deleteFrom('workflowRuns')

package/dist/src/kysely-workflow-service.d.ts

@@ -1,6 +1,6 @@
 import type { SerializedError } from '@pikku/core';
 import { PikkuWorkflowService, type WorkflowPlannedStep, type WorkflowQueueOptions, type WorkflowRun, type WorkflowRunWire, type StepState, type StepStatus, type WorkflowStatus, type WorkflowVersionStatus } from '@pikku/core/workflow';
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import type { KyselyPikkuDB } from './kysely-tables.js';
 export declare class KyselyWorkflowService extends PikkuWorkflowService {
     protected db: Kysely<KyselyPikkuDB>;
@@ -19,11 +19,26 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
         retryDelay?: string | number;
     }, fromStepName?: string): Promise<StepState>;
     getStepState(runId: string, stepName: string): Promise<StepState>;
+    protected listStepStates(runId: string): Promise<Array<StepState & {
+        stepName: string;
+    }>>;
     getRunHistory(runId: string): Promise<Array<StepState & {
         stepName: string;
     }>>;
     protected setStepRunningImpl(stepId: string): Promise<void>;
     protected setStepScheduledImpl(stepId: string): Promise<void>;
+    /**
+     * Move a step and its current history attempt to the same status in one
+     * transaction.
+     *
+     * Both halves matter: a crash between them used to leave the step row saying
+     * `succeeded` while its history row still said `running`, which silently
+     * corrupts `getRunHistory` and every timeline reconstruction built on it.
+     * The history row is targeted by its `attempt` rather than by a `LIMIT 1`
+     * over `created_at`, so a retry that lands in the same millisecond as the
+     * attempt it replaces still resolves the newer row.
+     */
+    private writeStepTransition;
     private insertHistoryRecord;
     private getTimestampFieldForStatus;
     protected setStepChildRunIdImpl(stepId: string, childRunId: string): Promise<void>;
@@ -37,7 +52,6 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
         failedNodeIds: string[];
         branchKeys: Record<string, string>;
     }>;
-    getNodesWithoutSteps(runId: string, nodeIds: string[]): Promise<string[]>;
     getStepInstances(runId: string): Promise<Array<{
         stepName: string;
         status: StepStatus;
@@ -45,6 +59,25 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
     }>>;
     getNodeResults(runId: string, nodeIds: string[]): Promise<Record<string, any>>;
     protected setBranchTakenImpl(stepId: string, branchKey: string): Promise<void>;
+    /**
+     * Merge one key into the run's JSON state, as a single expression evaluated
+     * by the database.
+     *
+     * The SQL is dialect-specific, so subclasses override this rather than the
+     * caller. The default is the SQLite form; MySQL and Postgres differ in how
+     * they cast a JSON literal.
+     *
+     * @param path - JSON path to the key, already quoted (`$."key"`)
+     * @param json - The value as JSON text
+     */
+    protected jsonSetState(path: string, json: string): import("kysely").RawBuilder<string>;
+    /**
+     * A JSON path for an arbitrary key. State keys are user-supplied (a graph's
+     * `setState` name, an approval's hex-encoded reason), so the key is quoted
+     * rather than interpolated bare — an unquoted `$.a.b` would silently address
+     * a nested path instead of the key literally called `a.b`.
+     */
+    private jsonPathFor;
     protected updateRunStateImpl(runId: string, name: string, value: unknown): Promise<void>;
     getRunState(runId: string): Promise<Record<string, unknown>>;
     protected upsertWorkflowVersionImpl(name: string, graphHash: string, graph: any, source: string, status?: WorkflowVersionStatus): Promise<void>;
@@ -53,10 +86,5 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
         graph: any;
         source: string;
     } | null>;
-    getAIGeneratedWorkflows(agentName?: string): Promise<Array<{
-        workflowName: string;
-        graphHash: string;
-        graph: any;
-    }>>;
     close(): Promise<void>;
 }