@pikku/kysely 0.13.6 → 0.13.7

package/src/kysely-scope-service.ts

@@ -1,5 +1,10 @@
 import type { Role, ScopeService } from '@pikku/core/services'
 import type { FlatScope } from '@pikku/core/scope'
+import type { SystemRole } from '@pikku/core/role'
+import {
+  SystemRoleImmutableError,
+  SystemRoleShadowedError,
+} from '@pikku/core/errors'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { ensurePikkuSchema } from './schema/index.js'
@@ -105,11 +110,80 @@ export class KyselyScopeService implements ScopeService {
     }))
   }
 
+  /**
+   * Registers the roles declared with `defineSystemRole`.
+   *
+   * Additive on the same terms as `syncScopes` — a role whose declaration has
+   * gone is marked `declared = false` rather than deleted, so a rollback or a
+   * rolling deploy cannot strip everyone's grant. Its *scope set* is replaced
+   * outright, because that is the declaration's entire content: editing
+   * `defineSystemRole` is how you change what a role means, and the deploy is
+   * when it takes effect.
+   */
+  async syncSystemRoles(roles: SystemRole[]): Promise<void> {
+    await this.db.transaction().execute(async (trx) => {
+      for (const role of roles) {
+        await trx
+          .insertInto('pikkuRoles')
+          .values({
+            name: role.name,
+            description: role.description ?? null,
+            system: true,
+            declared: true,
+          })
+          .onConflict((oc) =>
+            oc.column('name').doUpdateSet((eb) => ({
+              description: eb.ref('excluded.description'),
+              system: true,
+              declared: true,
+            }))
+          )
+          .execute()
+
+        await trx
+          .deleteFrom('pikkuRoleScopes')
+          .where('role', '=', role.name)
+          .execute()
+        if (role.scopes.length > 0) {
+          await trx
+            .insertInto('pikkuRoleScopes')
+            .values(role.scopes.map((scope) => ({ role: role.name, scope })))
+            .execute()
+        }
+      }
+
+      // Only system rows are in scope here: an admin's own role is not
+      // undeclared, it was never declared, and marking it would take it out of
+      // the console for a reason nobody could act on.
+      const markStale = trx
+        .updateTable('pikkuRoles')
+        .set({ declared: false })
+        .where('system', '=', true)
+      await (
+        roles.length > 0
+          ? markStale.where(
+              'name',
+              'not in',
+              roles.map((role) => role.name)
+            )
+          : markStale
+      ).execute()
+    })
+  }
+
   async createRole(role: Role): Promise<void> {
+    if (await this.isSystemRole(role.name)) {
+      throw new SystemRoleShadowedError(role.name)
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx
         .insertInto('pikkuRoles')
-        .values({ name: role.name, description: role.description ?? null })
+        .values({
+          name: role.name,
+          description: role.description ?? null,
+          system: false,
+          declared: true,
+        })
         .execute()
 
       if (role.scopes.length > 0) {
@@ -122,10 +196,16 @@ export class KyselyScopeService implements ScopeService {
   }
 
   async deleteRole(name: string): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 'delete')
+    }
     await this.db.deleteFrom('pikkuRoles').where('name', '=', name).execute()
   }
 
   async setRoleScopes(name: string, scopes: string[]): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 're-scope')
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx.deleteFrom('pikkuRoleScopes').where('role', '=', name).execute()
 
@@ -138,10 +218,26 @@ export class KyselyScopeService implements ScopeService {
     })
   }
 
+  /**
+   * Whether the store holds this name as a system role.
+   *
+   * Asked of the store rather than of the generated `SYSTEM_ROLES`, because the
+   * store is where a shadow would actually collide — and because a role whose
+   * declaration was deleted is still immutable until someone prunes it.
+   */
+  private async isSystemRole(name: string): Promise<boolean> {
+    const row = await this.db
+      .selectFrom('pikkuRoles')
+      .select('system')
+      .where('name', '=', name)
+      .executeTakeFirst()
+    return !!row?.system
+  }
+
   async listRoles(): Promise<Role[]> {
     const roles = await this.db
       .selectFrom('pikkuRoles')
-      .select(['name', 'description'])
+      .select(['name', 'description', 'system', 'declared'])
       .execute()
 
     const scopeRows = await this.db
@@ -163,6 +259,8 @@ export class KyselyScopeService implements ScopeService {
       name: role.name,
       description: role.description ?? undefined,
       scopes: byRole.get(role.name) ?? [],
+      system: !!role.system,
+      declared: !!role.declared,
     }))
   }
 
@@ -270,4 +368,55 @@ export class KyselyScopeService implements ScopeService {
 
     return deleted.map((r) => r.name)
   }
+
+  /**
+   * System roles the last sync found no declaration for, with how many people
+   * still hold each. Powers `pikku roles audit`.
+   *
+   * The count is the whole point: "this role is gone from the code" and "this
+   * role is gone from the code and 40 people are standing on it" call for
+   * different decisions, and the number is what makes pruning a choice rather
+   * than a formality.
+   */
+  async findStaleSystemRoles(): Promise<Array<{ role: string; users: number }>> {
+    const stale = await this.db
+      .selectFrom('pikkuRoles')
+      .leftJoin('pikkuUserRole', 'pikkuUserRole.role', 'pikkuRoles.name')
+      .select(['pikkuRoles.name as role', 'pikkuUserRole.userId as userId'])
+      .where('pikkuRoles.system', '=', true)
+      .where('pikkuRoles.declared', '=', false)
+      .execute()
+
+    const byRole = new Map<string, number>()
+    for (const row of stale) {
+      byRole.set(row.role, (byRole.get(row.role) ?? 0) + (row.userId ? 1 : 0))
+    }
+
+    return [...byRole.entries()].map(([role, users]) => ({ role, users }))
+  }
+
+  /**
+   * Removes undeclared system roles, cascading them out of every user grant
+   * that holds them. This revokes access, so it is never run implicitly.
+   */
+  async pruneSystemRoles(): Promise<string[]> {
+    const stale = await this.findStaleSystemRoles()
+    if (stale.length === 0) {
+      return []
+    }
+
+    const deleted = await this.db
+      .deleteFrom('pikkuRoles')
+      .where(
+        'name',
+        'in',
+        stale.map((s) => s.role)
+      )
+      .where('system', '=', true)
+      .where('declared', '=', false)
+      .returning('name')
+      .execute()
+
+    return deleted.map((r) => r.name)
+  }
 }

package/src/kysely-secret-service.ts

@@ -2,6 +2,8 @@ import type { SecretService } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import {
+  deriveKEK,
+  generateKEKSalt,
   envelopeEncrypt,
   envelopeDecrypt,
   envelopeRewrap,
@@ -19,6 +21,8 @@ export interface KyselySecretServiceConfig {
 
 export class KyselySecretService implements SecretService {
   private initialized = false
+  private kekSalts = new Map<number, string>()
+  private keks = new Map<number, CryptoKey>()
   private key: string
   private keyVersion: number
   private previousKey?: string
@@ -60,10 +64,52 @@ export class KyselySecretService implements SecretService {
       .execute()
   }
 
-  private getKEK(version: number): string {
-    if (version === this.keyVersion) return this.key
-    if (this.previousKey) return this.previousKey
-    throw new Error(`No KEK available for key_version ${version}`)
+  private async getKEKSalt(version: number): Promise<string> {
+    const cached = this.kekSalts.get(version)
+    if (cached) return cached
+
+    const existing = await this.db
+      .selectFrom('secretKekSalts')
+      .select('salt')
+      .where('keyVersion', '=', version)
+      .executeTakeFirst()
+
+    let salt = existing?.salt
+    if (!salt) {
+      await this.db
+        .insertInto('secretKekSalts')
+        .values({
+          keyVersion: version,
+          salt: generateKEKSalt(),
+          createdAt: new Date().toISOString() as unknown as Date,
+        })
+        .onConflict((oc) => oc.column('keyVersion').doNothing())
+        .execute()
+
+      const row = await this.db
+        .selectFrom('secretKekSalts')
+        .select('salt')
+        .where('keyVersion', '=', version)
+        .executeTakeFirstOrThrow()
+      salt = row.salt
+    }
+
+    this.kekSalts.set(version, salt)
+    return salt
+  }
+
+  private async getKEK(version: number): Promise<CryptoKey> {
+    const cached = this.keks.get(version)
+    if (cached) return cached
+
+    const passphrase = version === this.keyVersion ? this.key : this.previousKey
+    if (!passphrase) {
+      throw new Error(`No KEK available for key_version ${version}`)
+    }
+
+    const kek = await deriveKEK(passphrase, await this.getKEKSalt(version))
+    this.keks.set(version, kek)
+    return kek
   }
 
   async getSecret<T = string>(key: string): Promise<T> {
@@ -75,7 +121,7 @@ export class KyselySecretService implements SecretService {
 
     if (!row) throw new Error('Requested secret not found')
 
-    const kek = this.getKEK(row.keyVersion)
+    const kek = await this.getKEK(row.keyVersion)
     const result = await envelopeDecrypt<T>(kek, row.ciphertext, row.wrappedDek)
     await this.logAudit(key, 'read')
     return result
@@ -91,7 +137,10 @@ export class KyselySecretService implements SecretService {
   }
 
   async setSecret(key: string, value: unknown): Promise<void> {
-    const { ciphertext, wrappedDEK } = await envelopeEncrypt(this.key, value)
+    const { ciphertext, wrappedDEK } = await envelopeEncrypt(
+      await this.getKEK(this.keyVersion),
+      value
+    )
     const now = new Date().toISOString()
 
     await this.db
@@ -134,7 +183,7 @@ export class KyselySecretService implements SecretService {
     const out: Record<string, unknown> = {}
     for (const row of rows) {
       try {
-        const kek = this.getKEK(row.keyVersion)
+        const kek = await this.getKEK(row.keyVersion)
         out[row.key] = await envelopeDecrypt(
           kek,
           row.ciphertext,
@@ -168,12 +217,11 @@ export class KyselySecretService implements SecretService {
       .where('keyVersion', '<', this.keyVersion)
       .execute()
 
+    const oldKEK = await this.getKEK(this.keyVersion - 1)
+    const newKEK = await this.getKEK(this.keyVersion)
+
     for (const row of rows) {
-      const newWrappedDEK = await envelopeRewrap(
-        this.previousKey,
-        this.key,
-        row.wrappedDek
-      )
+      const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek)
       await this.db
         .updateTable('secrets')
         .set({

package/src/kysely-services.test.ts

@@ -1,4 +1,4 @@
-import { describe, test, before, after } from 'node:test'
+import { describe, test, before, beforeEach, after } from 'node:test'
 import assert from 'node:assert/strict'
 import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
 import { SerializePlugin } from './serialize-plugin.js'
@@ -54,8 +54,10 @@ async function dropAllTables(db: Kysely<KyselyPikkuDB>): Promise<void> {
     'workflow_runs',
     'workflow_versions',
     'secrets_audit',
+    'secret_kek_salts',
     'secrets',
     'credentials_audit',
+    'credential_kek_salts',
     'credentials',
   ]
   for (const table of tables) {
@@ -166,6 +168,126 @@ function registerTests(
     })
   })
 
+  describe(`KyselySecretService KEK derivation [${dialectName}]`, () => {
+    const kek = 'test-key-encryption-key-32chars!'
+
+    beforeEach(async () => {
+      await getDb().deleteFrom('secrets').execute()
+      await getDb().deleteFrom('secretKekSalts').execute()
+    })
+
+    test('a KEK salt row is created once per key version', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+      await service.setSecret('salt-a', 'one')
+      await service.setSecret('salt-b', 'two')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .select(['keyVersion', 'salt'])
+        .execute()
+
+      assert.equal(rows.length, 1)
+      assert.equal(rows[0]!.keyVersion, 1)
+      assert.ok(rows[0]!.salt.length > 0)
+    })
+
+    test('a second instance reuses the stored salt rather than minting one', async () => {
+      const first = new KyselySecretService(getDb(), { key: kek })
+      await first.init()
+      await first.setSecret('shared-salt', 'value')
+
+      const second = new KyselySecretService(getDb(), { key: kek })
+      await second.init()
+      assert.equal(await second.getSecret('shared-salt'), 'value')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .selectAll()
+        .execute()
+      assert.equal(rows.length, 1)
+    })
+
+    test('getSecrets over 50 secrets costs one KEK derivation, not 50', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+
+      const keys: string[] = []
+      for (let i = 0; i < 50; i++) {
+        const key = `bulk-${i}`
+        keys.push(key)
+        await service.setSecret(key, `value-${i}`)
+      }
+
+      const reader = new KyselySecretService(getDb(), { key: kek })
+      await reader.init()
+
+      const start = performance.now()
+      const out = await reader.getSecrets(keys)
+      const elapsed = performance.now() - start
+
+      assert.equal(Object.keys(out).length, 50)
+      assert.equal(out['bulk-7'], 'value-7')
+      assert.ok(
+        elapsed < 400,
+        `getSecrets over 50 secrets took ${elapsed.toFixed(0)}ms; one derivation per secret would cost seconds`
+      )
+    })
+
+    test('rotateKEK over 50 secrets derives two KEKs, not 100', async () => {
+      const previousKey = 'previous-key-encryption-key-32c!'
+      const writer = new KyselySecretService(getDb(), {
+        key: previousKey,
+        keyVersion: 1,
+      })
+      await writer.init()
+
+      for (let i = 0; i < 50; i++) {
+        await writer.setSecret(`rotate-${i}`, `value-${i}`)
+      }
+
+      const rotator = new KyselySecretService(getDb(), {
+        key: kek,
+        keyVersion: 2,
+        previousKey,
+      })
+      await rotator.init()
+
+      const start = performance.now()
+      const rotated = await rotator.rotateKEK()
+      const elapsed = performance.now() - start
+
+      assert.equal(rotated, 50)
+      assert.equal(await rotator.getSecret('rotate-7'), 'value-7')
+      assert.ok(
+        elapsed < 700,
+        `rotating 50 secrets took ${elapsed.toFixed(0)}ms; two derivations per secret would cost seconds`
+      )
+
+      const salts = await getDb()
+        .selectFrom('secretKekSalts')
+        .select('keyVersion')
+        .orderBy('keyVersion', 'asc')
+        .execute()
+      assert.deepEqual(
+        salts.map((r) => r.keyVersion),
+        [1, 2]
+      )
+    })
+
+    test('a wrong KEK passphrase fails loudly rather than returning garbage', async () => {
+      const writer = new KyselySecretService(getDb(), { key: kek })
+      await writer.init()
+      await writer.setSecret('wrong-kek', 'value')
+
+      const reader = new KyselySecretService(getDb(), {
+        key: 'a-completely-different-kek-32ch!',
+      })
+      await reader.init()
+      await assert.rejects(() => reader.getSecret('wrong-kek'))
+    })
+  })
+
   describe(`KyselyCredentialService audit [${dialectName}]`, () => {
     const kek = 'test-key-encryption-key-32chars!'
 

package/src/kysely-tables.ts

@@ -131,7 +131,14 @@ export interface AIRunTable {
   agentName: string
   threadId: string
   resourceId: string
-  status: Generated<'running' | 'suspended' | 'completed' | 'failed'>
+  /**
+   * `interrupted` is distinct from `failed`: the run was stopped on purpose,
+   * usually because the user talked over it. Nothing went wrong, so it should
+   * not show up in error reporting.
+   */
+  status: Generated<
+    'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'
+  >
   errorMessage: string | null
   suspendReason: 'approval' | 'credential' | 'rpc-missing' | null
   missingRpcs: string | null
@@ -165,6 +172,12 @@ export interface SecretsTable {
   updatedAt: Generated<Date>
 }
 
+export interface KekSaltsTable {
+  keyVersion: number
+  salt: string
+  createdAt: Generated<Date>
+}
+
 export interface SecretsAuditTable {
   id: string
   secretKey: string
@@ -215,10 +228,14 @@ export interface PikkuScopesTable {
   declared: Generated<boolean>
 }
 
-/** A role: an admin-composed bag of scopes. */
+/** A role: a bag of scopes, composed by an admin or declared in code. */
 export interface PikkuRolesTable {
   name: string
   description: string | null
+  /** Declared with `defineSystemRole`, and therefore not editable here. */
+  system: Generated<boolean>
+  /** False for a system role whose declaration has gone: held, but inert. */
+  declared: Generated<boolean>
   createdAt: Generated<Date>
 }
 
@@ -284,8 +301,10 @@ export interface KyselyPikkuDB {
   pikkuDeployments: PikkuDeploymentsTable
   pikkuDeploymentFunctions: PikkuDeploymentFunctionsTable
   secrets: SecretsTable
+  secretKekSalts: KekSaltsTable
   secretsAudit: SecretsAuditTable
   credentials: CredentialsTable
+  credentialKekSalts: KekSaltsTable
   credentialsAudit: CredentialsAuditTable
   pikkuUserSessions: UserSessionsTable
   webhookDelivery: WebhookDeliveryTable

package/src/kysely-workflow-service.ts

@@ -505,23 +505,6 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return { completedNodeIds, failedNodeIds, branchKeys }
   }
 
-  async getNodesWithoutSteps(
-    runId: string,
-    nodeIds: string[]
-  ): Promise<string[]> {
-    if (nodeIds.length === 0) return []
-
-    const result = await this.db
-      .selectFrom('workflowStep')
-      .select('stepName')
-      .where('workflowRunId', '=', runId)
-      .where('stepName', 'in', nodeIds)
-      .execute()
-
-    const existingStepNames = new Set(result.map((r) => r.stepName))
-    return nodeIds.filter((id) => !existingStepNames.has(id))
-  }
-
   async getStepInstances(
     runId: string
   ): Promise<

package/src/schema/credential.schema.ts

@@ -32,6 +32,15 @@ export const credentialSchema: PikkuSchema = {
       sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`
     ),
 
+    (db) =>
+      db.schema
+        .createTable('credentialKekSalts')
+        .addColumn('keyVersion', 'integer', (col) => col.primaryKey())
+        .addColumn('salt', 'varchar(64)', (col) => col.notNull())
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
     (db) =>
       db.schema
         .createTable('credentialsAudit')

package/src/schema/scope.schema.ts

@@ -35,6 +35,20 @@ export const scopeSchema: PikkuSchema = {
         .createTable('pikkuRoles')
         .addColumn('name', 'text', (col) => col.primaryKey())
         .addColumn('description', 'text')
+        // Declared in code with `defineSystemRole` rather than composed by an
+        // admin. What it buys is the refusals: a system role cannot be renamed,
+        // re-scoped or deleted from the console, and a console role cannot be
+        // created with its name — two rows answering to one name would make
+        // "does Susan hold `buyer`?" depend on which one the store returned.
+        .addColumn('system', 'boolean', (col) => col.defaultTo(false).notNull())
+        // A system role whose declaration has gone: still held by whoever holds
+        // it, no longer offered for new grants, awaiting `pikku roles prune`.
+        // The same additive contract as `pikkuScopes.declared`, and for the same
+        // reason — a mid-deploy revocation is not something a code edit should
+        // be able to cause.
+        .addColumn('declared', 'boolean', (col) =>
+          col.defaultTo(true).notNull()
+        )
         .addColumn('createdAt', 'timestamp', (col) =>
           col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
         ),

package/src/schema/secret.schema.ts

@@ -27,6 +27,15 @@ export const secretSchema: PikkuSchema = {
           col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
         ),
 
+    (db) =>
+      db.schema
+        .createTable('secretKekSalts')
+        .addColumn('keyVersion', 'integer', (col) => col.primaryKey())
+        .addColumn('salt', 'varchar(64)', (col) => col.notNull())
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
     (db) =>
       db.schema
         .createTable('secretsAudit')