@pikku/kysely 0.13.5 → 0.13.7

package/src/kysely-credential-service.ts

@@ -2,6 +2,8 @@ import type { CredentialService } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import {
+  deriveKEK,
+  generateKEKSalt,
   envelopeEncrypt,
   envelopeDecrypt,
   envelopeRewrap,
@@ -19,6 +21,8 @@ export interface KyselyCredentialServiceConfig {
 
 export class KyselyCredentialService implements CredentialService {
   private initialized = false
+  private kekSalts = new Map<number, string>()
+  private keks = new Map<number, CryptoKey>()
   private key: string
   private keyVersion: number
   private previousKey?: string
@@ -62,10 +66,52 @@ export class KyselyCredentialService implements CredentialService {
       .execute()
   }
 
-  private getKEK(version: number): string {
-    if (version === this.keyVersion) return this.key
-    if (this.previousKey) return this.previousKey
-    throw new Error(`No KEK available for key_version ${version}`)
+  private async getKEKSalt(version: number): Promise<string> {
+    const cached = this.kekSalts.get(version)
+    if (cached) return cached
+
+    const existing = await this.db
+      .selectFrom('credentialKekSalts')
+      .select('salt')
+      .where('keyVersion', '=', version)
+      .executeTakeFirst()
+
+    let salt = existing?.salt
+    if (!salt) {
+      await this.db
+        .insertInto('credentialKekSalts')
+        .values({
+          keyVersion: version,
+          salt: generateKEKSalt(),
+          createdAt: new Date().toISOString() as unknown as Date,
+        })
+        .onConflict((oc) => oc.column('keyVersion').doNothing())
+        .execute()
+
+      const row = await this.db
+        .selectFrom('credentialKekSalts')
+        .select('salt')
+        .where('keyVersion', '=', version)
+        .executeTakeFirstOrThrow()
+      salt = row.salt
+    }
+
+    this.kekSalts.set(version, salt)
+    return salt
+  }
+
+  private async getKEK(version: number): Promise<CryptoKey> {
+    const cached = this.keks.get(version)
+    if (cached) return cached
+
+    const passphrase = version === this.keyVersion ? this.key : this.previousKey
+    if (!passphrase) {
+      throw new Error(`No KEK available for key_version ${version}`)
+    }
+
+    const kek = await deriveKEK(passphrase, await this.getKEKSalt(version))
+    this.keks.set(version, kek)
+    return kek
   }
 
   private whereUserId(qb: any, userId?: string) {
@@ -84,7 +130,7 @@ export class KyselyCredentialService implements CredentialService {
     const row = await qb.executeTakeFirst()
     if (!row) return null
 
-    const kek = this.getKEK(row.keyVersion)
+    const kek = await this.getKEK(row.keyVersion)
     const plaintext = await envelopeDecrypt<string>(
       kek,
       row.ciphertext,
@@ -102,7 +148,7 @@ export class KyselyCredentialService implements CredentialService {
   async set(name: string, value: unknown, userId?: string): Promise<void> {
     const plaintext = JSON.stringify(value)
     const { ciphertext, wrappedDEK } = await envelopeEncrypt(
-      this.key,
+      await this.getKEK(this.keyVersion),
       plaintext
     )
     const now = new Date().toISOString()
@@ -166,7 +212,7 @@ export class KyselyCredentialService implements CredentialService {
 
     const result: Record<string, unknown> = {}
     for (const row of rows) {
-      const kek = this.getKEK(row.keyVersion)
+      const kek = await this.getKEK(row.keyVersion)
       const plaintext = await envelopeDecrypt<string>(
         kek,
         row.ciphertext,
@@ -216,12 +262,11 @@ export class KyselyCredentialService implements CredentialService {
       .where('keyVersion', '<', this.keyVersion)
       .execute()
 
+    const oldKEK = await this.getKEK(this.keyVersion - 1)
+    const newKEK = await this.getKEK(this.keyVersion)
+
     for (const row of rows) {
-      const newWrappedDEK = await envelopeRewrap(
-        this.previousKey,
-        this.key,
-        row.wrappedDek
-      )
+      const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek)
       let qb = this.db
         .updateTable('credentials')
         .set({

package/src/kysely-scope-service.ts

@@ -1,5 +1,10 @@
 import type { Role, ScopeService } from '@pikku/core/services'
 import type { FlatScope } from '@pikku/core/scope'
+import type { SystemRole } from '@pikku/core/role'
+import {
+  SystemRoleImmutableError,
+  SystemRoleShadowedError,
+} from '@pikku/core/errors'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { ensurePikkuSchema } from './schema/index.js'
@@ -105,11 +110,80 @@ export class KyselyScopeService implements ScopeService {
     }))
   }
 
+  /**
+   * Registers the roles declared with `defineSystemRole`.
+   *
+   * Additive on the same terms as `syncScopes` — a role whose declaration has
+   * gone is marked `declared = false` rather than deleted, so a rollback or a
+   * rolling deploy cannot strip everyone's grant. Its *scope set* is replaced
+   * outright, because that is the declaration's entire content: editing
+   * `defineSystemRole` is how you change what a role means, and the deploy is
+   * when it takes effect.
+   */
+  async syncSystemRoles(roles: SystemRole[]): Promise<void> {
+    await this.db.transaction().execute(async (trx) => {
+      for (const role of roles) {
+        await trx
+          .insertInto('pikkuRoles')
+          .values({
+            name: role.name,
+            description: role.description ?? null,
+            system: true,
+            declared: true,
+          })
+          .onConflict((oc) =>
+            oc.column('name').doUpdateSet((eb) => ({
+              description: eb.ref('excluded.description'),
+              system: true,
+              declared: true,
+            }))
+          )
+          .execute()
+
+        await trx
+          .deleteFrom('pikkuRoleScopes')
+          .where('role', '=', role.name)
+          .execute()
+        if (role.scopes.length > 0) {
+          await trx
+            .insertInto('pikkuRoleScopes')
+            .values(role.scopes.map((scope) => ({ role: role.name, scope })))
+            .execute()
+        }
+      }
+
+      // Only system rows are in scope here: an admin's own role is not
+      // undeclared, it was never declared, and marking it would take it out of
+      // the console for a reason nobody could act on.
+      const markStale = trx
+        .updateTable('pikkuRoles')
+        .set({ declared: false })
+        .where('system', '=', true)
+      await (
+        roles.length > 0
+          ? markStale.where(
+              'name',
+              'not in',
+              roles.map((role) => role.name)
+            )
+          : markStale
+      ).execute()
+    })
+  }
+
   async createRole(role: Role): Promise<void> {
+    if (await this.isSystemRole(role.name)) {
+      throw new SystemRoleShadowedError(role.name)
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx
         .insertInto('pikkuRoles')
-        .values({ name: role.name, description: role.description ?? null })
+        .values({
+          name: role.name,
+          description: role.description ?? null,
+          system: false,
+          declared: true,
+        })
         .execute()
 
       if (role.scopes.length > 0) {
@@ -122,10 +196,16 @@ export class KyselyScopeService implements ScopeService {
   }
 
   async deleteRole(name: string): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 'delete')
+    }
     await this.db.deleteFrom('pikkuRoles').where('name', '=', name).execute()
   }
 
   async setRoleScopes(name: string, scopes: string[]): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 're-scope')
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx.deleteFrom('pikkuRoleScopes').where('role', '=', name).execute()
 
@@ -138,10 +218,26 @@ export class KyselyScopeService implements ScopeService {
     })
   }
 
+  /**
+   * Whether the store holds this name as a system role.
+   *
+   * Asked of the store rather than of the generated `SYSTEM_ROLES`, because the
+   * store is where a shadow would actually collide — and because a role whose
+   * declaration was deleted is still immutable until someone prunes it.
+   */
+  private async isSystemRole(name: string): Promise<boolean> {
+    const row = await this.db
+      .selectFrom('pikkuRoles')
+      .select('system')
+      .where('name', '=', name)
+      .executeTakeFirst()
+    return !!row?.system
+  }
+
   async listRoles(): Promise<Role[]> {
     const roles = await this.db
       .selectFrom('pikkuRoles')
-      .select(['name', 'description'])
+      .select(['name', 'description', 'system', 'declared'])
       .execute()
 
     const scopeRows = await this.db
@@ -163,6 +259,8 @@ export class KyselyScopeService implements ScopeService {
       name: role.name,
       description: role.description ?? undefined,
       scopes: byRole.get(role.name) ?? [],
+      system: !!role.system,
+      declared: !!role.declared,
     }))
   }
 
@@ -270,4 +368,55 @@ export class KyselyScopeService implements ScopeService {
 
     return deleted.map((r) => r.name)
   }
+
+  /**
+   * System roles the last sync found no declaration for, with how many people
+   * still hold each. Powers `pikku roles audit`.
+   *
+   * The count is the whole point: "this role is gone from the code" and "this
+   * role is gone from the code and 40 people are standing on it" call for
+   * different decisions, and the number is what makes pruning a choice rather
+   * than a formality.
+   */
+  async findStaleSystemRoles(): Promise<Array<{ role: string; users: number }>> {
+    const stale = await this.db
+      .selectFrom('pikkuRoles')
+      .leftJoin('pikkuUserRole', 'pikkuUserRole.role', 'pikkuRoles.name')
+      .select(['pikkuRoles.name as role', 'pikkuUserRole.userId as userId'])
+      .where('pikkuRoles.system', '=', true)
+      .where('pikkuRoles.declared', '=', false)
+      .execute()
+
+    const byRole = new Map<string, number>()
+    for (const row of stale) {
+      byRole.set(row.role, (byRole.get(row.role) ?? 0) + (row.userId ? 1 : 0))
+    }
+
+    return [...byRole.entries()].map(([role, users]) => ({ role, users }))
+  }
+
+  /**
+   * Removes undeclared system roles, cascading them out of every user grant
+   * that holds them. This revokes access, so it is never run implicitly.
+   */
+  async pruneSystemRoles(): Promise<string[]> {
+    const stale = await this.findStaleSystemRoles()
+    if (stale.length === 0) {
+      return []
+    }
+
+    const deleted = await this.db
+      .deleteFrom('pikkuRoles')
+      .where(
+        'name',
+        'in',
+        stale.map((s) => s.role)
+      )
+      .where('system', '=', true)
+      .where('declared', '=', false)
+      .returning('name')
+      .execute()
+
+    return deleted.map((r) => r.name)
+  }
 }

package/src/kysely-secret-service.ts

@@ -2,6 +2,8 @@ import type { SecretService } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import {
+  deriveKEK,
+  generateKEKSalt,
   envelopeEncrypt,
   envelopeDecrypt,
   envelopeRewrap,
@@ -19,6 +21,8 @@ export interface KyselySecretServiceConfig {
 
 export class KyselySecretService implements SecretService {
   private initialized = false
+  private kekSalts = new Map<number, string>()
+  private keks = new Map<number, CryptoKey>()
   private key: string
   private keyVersion: number
   private previousKey?: string
@@ -60,10 +64,52 @@ export class KyselySecretService implements SecretService {
       .execute()
   }
 
-  private getKEK(version: number): string {
-    if (version === this.keyVersion) return this.key
-    if (this.previousKey) return this.previousKey
-    throw new Error(`No KEK available for key_version ${version}`)
+  private async getKEKSalt(version: number): Promise<string> {
+    const cached = this.kekSalts.get(version)
+    if (cached) return cached
+
+    const existing = await this.db
+      .selectFrom('secretKekSalts')
+      .select('salt')
+      .where('keyVersion', '=', version)
+      .executeTakeFirst()
+
+    let salt = existing?.salt
+    if (!salt) {
+      await this.db
+        .insertInto('secretKekSalts')
+        .values({
+          keyVersion: version,
+          salt: generateKEKSalt(),
+          createdAt: new Date().toISOString() as unknown as Date,
+        })
+        .onConflict((oc) => oc.column('keyVersion').doNothing())
+        .execute()
+
+      const row = await this.db
+        .selectFrom('secretKekSalts')
+        .select('salt')
+        .where('keyVersion', '=', version)
+        .executeTakeFirstOrThrow()
+      salt = row.salt
+    }
+
+    this.kekSalts.set(version, salt)
+    return salt
+  }
+
+  private async getKEK(version: number): Promise<CryptoKey> {
+    const cached = this.keks.get(version)
+    if (cached) return cached
+
+    const passphrase = version === this.keyVersion ? this.key : this.previousKey
+    if (!passphrase) {
+      throw new Error(`No KEK available for key_version ${version}`)
+    }
+
+    const kek = await deriveKEK(passphrase, await this.getKEKSalt(version))
+    this.keks.set(version, kek)
+    return kek
   }
 
   async getSecret<T = string>(key: string): Promise<T> {
@@ -75,7 +121,7 @@ export class KyselySecretService implements SecretService {
 
     if (!row) throw new Error('Requested secret not found')
 
-    const kek = this.getKEK(row.keyVersion)
+    const kek = await this.getKEK(row.keyVersion)
     const result = await envelopeDecrypt<T>(kek, row.ciphertext, row.wrappedDek)
     await this.logAudit(key, 'read')
     return result
@@ -91,7 +137,10 @@ export class KyselySecretService implements SecretService {
   }
 
   async setSecret(key: string, value: unknown): Promise<void> {
-    const { ciphertext, wrappedDEK } = await envelopeEncrypt(this.key, value)
+    const { ciphertext, wrappedDEK } = await envelopeEncrypt(
+      await this.getKEK(this.keyVersion),
+      value
+    )
     const now = new Date().toISOString()
 
     await this.db
@@ -134,7 +183,7 @@ export class KyselySecretService implements SecretService {
     const out: Record<string, unknown> = {}
     for (const row of rows) {
       try {
-        const kek = this.getKEK(row.keyVersion)
+        const kek = await this.getKEK(row.keyVersion)
         out[row.key] = await envelopeDecrypt(
           kek,
           row.ciphertext,
@@ -168,12 +217,11 @@ export class KyselySecretService implements SecretService {
       .where('keyVersion', '<', this.keyVersion)
       .execute()
 
+    const oldKEK = await this.getKEK(this.keyVersion - 1)
+    const newKEK = await this.getKEK(this.keyVersion)
+
     for (const row of rows) {
-      const newWrappedDEK = await envelopeRewrap(
-        this.previousKey,
-        this.key,
-        row.wrappedDek
-      )
+      const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek)
       await this.db
         .updateTable('secrets')
         .set({

package/src/kysely-services.test.ts

@@ -1,4 +1,4 @@
-import { describe, test, before, after } from 'node:test'
+import { describe, test, before, beforeEach, after } from 'node:test'
 import assert from 'node:assert/strict'
 import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
 import { SerializePlugin } from './serialize-plugin.js'
@@ -54,8 +54,10 @@ async function dropAllTables(db: Kysely<KyselyPikkuDB>): Promise<void> {
     'workflow_runs',
     'workflow_versions',
     'secrets_audit',
+    'secret_kek_salts',
     'secrets',
     'credentials_audit',
+    'credential_kek_salts',
     'credentials',
   ]
   for (const table of tables) {
@@ -166,6 +168,126 @@ function registerTests(
     })
   })
 
+  describe(`KyselySecretService KEK derivation [${dialectName}]`, () => {
+    const kek = 'test-key-encryption-key-32chars!'
+
+    beforeEach(async () => {
+      await getDb().deleteFrom('secrets').execute()
+      await getDb().deleteFrom('secretKekSalts').execute()
+    })
+
+    test('a KEK salt row is created once per key version', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+      await service.setSecret('salt-a', 'one')
+      await service.setSecret('salt-b', 'two')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .select(['keyVersion', 'salt'])
+        .execute()
+
+      assert.equal(rows.length, 1)
+      assert.equal(rows[0]!.keyVersion, 1)
+      assert.ok(rows[0]!.salt.length > 0)
+    })
+
+    test('a second instance reuses the stored salt rather than minting one', async () => {
+      const first = new KyselySecretService(getDb(), { key: kek })
+      await first.init()
+      await first.setSecret('shared-salt', 'value')
+
+      const second = new KyselySecretService(getDb(), { key: kek })
+      await second.init()
+      assert.equal(await second.getSecret('shared-salt'), 'value')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .selectAll()
+        .execute()
+      assert.equal(rows.length, 1)
+    })
+
+    test('getSecrets over 50 secrets costs one KEK derivation, not 50', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+
+      const keys: string[] = []
+      for (let i = 0; i < 50; i++) {
+        const key = `bulk-${i}`
+        keys.push(key)
+        await service.setSecret(key, `value-${i}`)
+      }
+
+      const reader = new KyselySecretService(getDb(), { key: kek })
+      await reader.init()
+
+      const start = performance.now()
+      const out = await reader.getSecrets(keys)
+      const elapsed = performance.now() - start
+
+      assert.equal(Object.keys(out).length, 50)
+      assert.equal(out['bulk-7'], 'value-7')
+      assert.ok(
+        elapsed < 400,
+        `getSecrets over 50 secrets took ${elapsed.toFixed(0)}ms; one derivation per secret would cost seconds`
+      )
+    })
+
+    test('rotateKEK over 50 secrets derives two KEKs, not 100', async () => {
+      const previousKey = 'previous-key-encryption-key-32c!'
+      const writer = new KyselySecretService(getDb(), {
+        key: previousKey,
+        keyVersion: 1,
+      })
+      await writer.init()
+
+      for (let i = 0; i < 50; i++) {
+        await writer.setSecret(`rotate-${i}`, `value-${i}`)
+      }
+
+      const rotator = new KyselySecretService(getDb(), {
+        key: kek,
+        keyVersion: 2,
+        previousKey,
+      })
+      await rotator.init()
+
+      const start = performance.now()
+      const rotated = await rotator.rotateKEK()
+      const elapsed = performance.now() - start
+
+      assert.equal(rotated, 50)
+      assert.equal(await rotator.getSecret('rotate-7'), 'value-7')
+      assert.ok(
+        elapsed < 700,
+        `rotating 50 secrets took ${elapsed.toFixed(0)}ms; two derivations per secret would cost seconds`
+      )
+
+      const salts = await getDb()
+        .selectFrom('secretKekSalts')
+        .select('keyVersion')
+        .orderBy('keyVersion', 'asc')
+        .execute()
+      assert.deepEqual(
+        salts.map((r) => r.keyVersion),
+        [1, 2]
+      )
+    })
+
+    test('a wrong KEK passphrase fails loudly rather than returning garbage', async () => {
+      const writer = new KyselySecretService(getDb(), { key: kek })
+      await writer.init()
+      await writer.setSecret('wrong-kek', 'value')
+
+      const reader = new KyselySecretService(getDb(), {
+        key: 'a-completely-different-kek-32ch!',
+      })
+      await reader.init()
+      await assert.rejects(() => reader.getSecret('wrong-kek'))
+    })
+  })
+
   describe(`KyselyCredentialService audit [${dialectName}]`, () => {
     const kek = 'test-key-encryption-key-32chars!'
 

package/src/kysely-tables.ts

@@ -52,6 +52,8 @@ export interface WorkflowStepTable {
   retries: number | null
   retryDelay: string | null
   fromStepName: string | null
+  /** The history attempt currently in flight; see WorkflowStepHistoryTable.attempt. */
+  currentAttempt: number | null
   createdAt: Generated<Date>
   updatedAt: Generated<Date>
 }
@@ -62,6 +64,12 @@ export interface WorkflowStepHistoryTable {
   status: StepStatus
   result: string | null
   error: string | null
+  /**
+   * Monotonic attempt number within the step, starting at 1. Orders attempts
+   * and identifies the live one without relying on `createdAt`, which cannot
+   * separate a retry from the attempt it replaces in the same millisecond.
+   */
+  attempt: number | null
   createdAt: Generated<Date>
   runningAt: Date | null
   scheduledAt: Date | null
@@ -123,7 +131,14 @@ export interface AIRunTable {
   agentName: string
   threadId: string
   resourceId: string
-  status: Generated<'running' | 'suspended' | 'completed' | 'failed'>
+  /**
+   * `interrupted` is distinct from `failed`: the run was stopped on purpose,
+   * usually because the user talked over it. Nothing went wrong, so it should
+   * not show up in error reporting.
+   */
+  status: Generated<
+    'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'
+  >
   errorMessage: string | null
   suspendReason: 'approval' | 'credential' | 'rpc-missing' | null
   missingRpcs: string | null
@@ -157,6 +172,12 @@ export interface SecretsTable {
   updatedAt: Generated<Date>
 }
 
+export interface KekSaltsTable {
+  keyVersion: number
+  salt: string
+  createdAt: Generated<Date>
+}
+
 export interface SecretsAuditTable {
   id: string
   secretKey: string
@@ -207,10 +228,14 @@ export interface PikkuScopesTable {
   declared: Generated<boolean>
 }
 
-/** A role: an admin-composed bag of scopes. */
+/** A role: a bag of scopes, composed by an admin or declared in code. */
 export interface PikkuRolesTable {
   name: string
   description: string | null
+  /** Declared with `defineSystemRole`, and therefore not editable here. */
+  system: Generated<boolean>
+  /** False for a system role whose declaration has gone: held, but inert. */
+  declared: Generated<boolean>
   createdAt: Generated<Date>
 }
 
@@ -276,8 +301,10 @@ export interface KyselyPikkuDB {
   pikkuDeployments: PikkuDeploymentsTable
   pikkuDeploymentFunctions: PikkuDeploymentFunctionsTable
   secrets: SecretsTable
+  secretKekSalts: KekSaltsTable
   secretsAudit: SecretsAuditTable
   credentials: CredentialsTable
+  credentialKekSalts: KekSaltsTable
   credentialsAudit: CredentialsAuditTable
   pikkuUserSessions: UserSessionsTable
   webhookDelivery: WebhookDeliveryTable

package/src/kysely-workflow-mirror.test.ts

@@ -1,6 +1,6 @@
 import { describe, test, beforeEach, afterEach } from 'node:test'
 import assert from 'node:assert/strict'
-import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
+import { CamelCasePlugin, Kysely, SqliteDialect, sql } from 'kysely'
 import { SerializePlugin } from './serialize-plugin.js'
 import Database from 'better-sqlite3'
 
@@ -237,18 +237,6 @@ describe('KyselyWorkflowMirror — workflow versions', () => {
       .executeTakeFirst()
     assert.equal(row?.status, 'archived')
   })
-
-  test('AI-generated workflows surface via getAIGeneratedWorkflows', async () => {
-    await mirror.upsertWorkflowVersion(
-      'ai:agent-x:foo',
-      'h1',
-      { nodes: [] },
-      'dynamic-workflow'
-    )
-    const list = await runService.getAIGeneratedWorkflows('agent-x')
-    assert.equal(list.length, 1)
-    assert.equal(list[0]!.workflowName, 'ai:agent-x:foo')
-  })
 })
 
 describe('KyselyWorkflowMirror — init', () => {