@pikku/kysely 0.13.6 → 0.13.10

package/src/kysely-scope-service.ts

@@ -1,5 +1,10 @@
 import type { Role, ScopeService } from '@pikku/core/services'
 import type { FlatScope } from '@pikku/core/scope'
+import type { SystemRole } from '@pikku/core/role'
+import {
+  SystemRoleImmutableError,
+  SystemRoleShadowedError,
+} from '@pikku/core/errors'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { ensurePikkuSchema } from './schema/index.js'
@@ -105,11 +110,80 @@ export class KyselyScopeService implements ScopeService {
     }))
   }
 
+  /**
+   * Registers the roles declared with `defineSystemRole`.
+   *
+   * Additive on the same terms as `syncScopes` — a role whose declaration has
+   * gone is marked `declared = false` rather than deleted, so a rollback or a
+   * rolling deploy cannot strip everyone's grant. Its *scope set* is replaced
+   * outright, because that is the declaration's entire content: editing
+   * `defineSystemRole` is how you change what a role means, and the deploy is
+   * when it takes effect.
+   */
+  async syncSystemRoles(roles: SystemRole[]): Promise<void> {
+    await this.db.transaction().execute(async (trx) => {
+      for (const role of roles) {
+        await trx
+          .insertInto('pikkuRoles')
+          .values({
+            name: role.name,
+            description: role.description ?? null,
+            system: true,
+            declared: true,
+          })
+          .onConflict((oc) =>
+            oc.column('name').doUpdateSet((eb) => ({
+              description: eb.ref('excluded.description'),
+              system: true,
+              declared: true,
+            }))
+          )
+          .execute()
+
+        await trx
+          .deleteFrom('pikkuRoleScopes')
+          .where('role', '=', role.name)
+          .execute()
+        if (role.scopes.length > 0) {
+          await trx
+            .insertInto('pikkuRoleScopes')
+            .values(role.scopes.map((scope) => ({ role: role.name, scope })))
+            .execute()
+        }
+      }
+
+      // Only system rows are in scope here: an admin's own role is not
+      // undeclared, it was never declared, and marking it would take it out of
+      // the console for a reason nobody could act on.
+      const markStale = trx
+        .updateTable('pikkuRoles')
+        .set({ declared: false })
+        .where('system', '=', true)
+      await (
+        roles.length > 0
+          ? markStale.where(
+              'name',
+              'not in',
+              roles.map((role) => role.name)
+            )
+          : markStale
+      ).execute()
+    })
+  }
+
   async createRole(role: Role): Promise<void> {
+    if (await this.isSystemRole(role.name)) {
+      throw new SystemRoleShadowedError(role.name)
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx
         .insertInto('pikkuRoles')
-        .values({ name: role.name, description: role.description ?? null })
+        .values({
+          name: role.name,
+          description: role.description ?? null,
+          system: false,
+          declared: true,
+        })
         .execute()
 
       if (role.scopes.length > 0) {
@@ -122,10 +196,16 @@ export class KyselyScopeService implements ScopeService {
   }
 
   async deleteRole(name: string): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 'delete')
+    }
     await this.db.deleteFrom('pikkuRoles').where('name', '=', name).execute()
   }
 
   async setRoleScopes(name: string, scopes: string[]): Promise<void> {
+    if (await this.isSystemRole(name)) {
+      throw new SystemRoleImmutableError(name, 're-scope')
+    }
     await this.db.transaction().execute(async (trx) => {
       await trx.deleteFrom('pikkuRoleScopes').where('role', '=', name).execute()
 
@@ -138,10 +218,26 @@ export class KyselyScopeService implements ScopeService {
     })
   }
 
+  /**
+   * Whether the store holds this name as a system role.
+   *
+   * Asked of the store rather than of the generated `SYSTEM_ROLES`, because the
+   * store is where a shadow would actually collide — and because a role whose
+   * declaration was deleted is still immutable until someone prunes it.
+   */
+  private async isSystemRole(name: string): Promise<boolean> {
+    const row = await this.db
+      .selectFrom('pikkuRoles')
+      .select('system')
+      .where('name', '=', name)
+      .executeTakeFirst()
+    return !!row?.system
+  }
+
   async listRoles(): Promise<Role[]> {
     const roles = await this.db
       .selectFrom('pikkuRoles')
-      .select(['name', 'description'])
+      .select(['name', 'description', 'system', 'declared'])
       .execute()
 
     const scopeRows = await this.db
@@ -163,6 +259,8 @@ export class KyselyScopeService implements ScopeService {
       name: role.name,
       description: role.description ?? undefined,
       scopes: byRole.get(role.name) ?? [],
+      system: !!role.system,
+      declared: !!role.declared,
     }))
   }
 
@@ -270,4 +368,55 @@ export class KyselyScopeService implements ScopeService {
 
     return deleted.map((r) => r.name)
   }
+
+  /**
+   * System roles the last sync found no declaration for, with how many people
+   * still hold each. Powers `pikku roles audit`.
+   *
+   * The count is the whole point: "this role is gone from the code" and "this
+   * role is gone from the code and 40 people are standing on it" call for
+   * different decisions, and the number is what makes pruning a choice rather
+   * than a formality.
+   */
+  async findStaleSystemRoles(): Promise<Array<{ role: string; users: number }>> {
+    const stale = await this.db
+      .selectFrom('pikkuRoles')
+      .leftJoin('pikkuUserRole', 'pikkuUserRole.role', 'pikkuRoles.name')
+      .select(['pikkuRoles.name as role', 'pikkuUserRole.userId as userId'])
+      .where('pikkuRoles.system', '=', true)
+      .where('pikkuRoles.declared', '=', false)
+      .execute()
+
+    const byRole = new Map<string, number>()
+    for (const row of stale) {
+      byRole.set(row.role, (byRole.get(row.role) ?? 0) + (row.userId ? 1 : 0))
+    }
+
+    return [...byRole.entries()].map(([role, users]) => ({ role, users }))
+  }
+
+  /**
+   * Removes undeclared system roles, cascading them out of every user grant
+   * that holds them. This revokes access, so it is never run implicitly.
+   */
+  async pruneSystemRoles(): Promise<string[]> {
+    const stale = await this.findStaleSystemRoles()
+    if (stale.length === 0) {
+      return []
+    }
+
+    const deleted = await this.db
+      .deleteFrom('pikkuRoles')
+      .where(
+        'name',
+        'in',
+        stale.map((s) => s.role)
+      )
+      .where('system', '=', true)
+      .where('declared', '=', false)
+      .returning('name')
+      .execute()
+
+    return deleted.map((r) => r.name)
+  }
 }

package/src/kysely-secret-service.ts

@@ -1,7 +1,10 @@
-import type { SecretService } from '@pikku/core/services'
+import { createSecretValue, isSecretValue, type SecretValue } from '@pikku/core'
+import type { SecretService, SecretValues } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import {
+  deriveKEK,
+  generateKEKSalt,
   envelopeEncrypt,
   envelopeDecrypt,
   envelopeRewrap,
@@ -19,6 +22,8 @@ export interface KyselySecretServiceConfig {
 
 export class KyselySecretService implements SecretService {
   private initialized = false
+  private kekSalts = new Map<number, string>()
+  private keks = new Map<number, CryptoKey>()
   private key: string
   private keyVersion: number
   private previousKey?: string
@@ -60,13 +65,55 @@ export class KyselySecretService implements SecretService {
       .execute()
   }
 
-  private getKEK(version: number): string {
-    if (version === this.keyVersion) return this.key
-    if (this.previousKey) return this.previousKey
-    throw new Error(`No KEK available for key_version ${version}`)
+  private async getKEKSalt(version: number): Promise<string> {
+    const cached = this.kekSalts.get(version)
+    if (cached) return cached
+
+    const existing = await this.db
+      .selectFrom('secretKekSalts')
+      .select('salt')
+      .where('keyVersion', '=', version)
+      .executeTakeFirst()
+
+    let salt = existing?.salt
+    if (!salt) {
+      await this.db
+        .insertInto('secretKekSalts')
+        .values({
+          keyVersion: version,
+          salt: generateKEKSalt(),
+          createdAt: new Date().toISOString() as unknown as Date,
+        })
+        .onConflict((oc) => oc.column('keyVersion').doNothing())
+        .execute()
+
+      const row = await this.db
+        .selectFrom('secretKekSalts')
+        .select('salt')
+        .where('keyVersion', '=', version)
+        .executeTakeFirstOrThrow()
+      salt = row.salt
+    }
+
+    this.kekSalts.set(version, salt)
+    return salt
+  }
+
+  private async getKEK(version: number): Promise<CryptoKey> {
+    const cached = this.keks.get(version)
+    if (cached) return cached
+
+    const passphrase = version === this.keyVersion ? this.key : this.previousKey
+    if (!passphrase) {
+      throw new Error(`No KEK available for key_version ${version}`)
+    }
+
+    const kek = await deriveKEK(passphrase, await this.getKEKSalt(version))
+    this.keks.set(version, kek)
+    return kek
   }
 
-  async getSecret<T = string>(key: string): Promise<T> {
+  async getSecret<T = string>(key: string): Promise<SecretValue<T>> {
     const row = await this.db
       .selectFrom('secrets')
       .select(['ciphertext', 'wrappedDek', 'keyVersion'])
@@ -75,10 +122,10 @@ export class KyselySecretService implements SecretService {
 
     if (!row) throw new Error('Requested secret not found')
 
-    const kek = this.getKEK(row.keyVersion)
+    const kek = await this.getKEK(row.keyVersion)
     const result = await envelopeDecrypt<T>(kek, row.ciphertext, row.wrappedDek)
     await this.logAudit(key, 'read')
-    return result
+    return createSecretValue(result)
   }
 
   async hasSecret(key: string): Promise<boolean> {
@@ -91,7 +138,12 @@ export class KyselySecretService implements SecretService {
   }
 
   async setSecret(key: string, value: unknown): Promise<void> {
-    const { ciphertext, wrappedDEK } = await envelopeEncrypt(this.key, value)
+    // Encrypting the wrapper would store its redaction, not the secret —
+    // writing a secret back to the vault is exactly what this method is for.
+    const { ciphertext, wrappedDEK } = await envelopeEncrypt(
+      await this.getKEK(this.keyVersion),
+      isSecretValue(value) ? value.reveal() : value
+    )
     const now = new Date().toISOString()
 
     await this.db
@@ -124,7 +176,7 @@ export class KyselySecretService implements SecretService {
 
   async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
     keys: (keyof T & string)[]
-  ): Promise<T> {
+  ): Promise<Partial<SecretValues<T>>> {
     const rows = await this.db
       .selectFrom('secrets')
       .select(['key', 'ciphertext', 'wrappedDek', 'keyVersion'])
@@ -134,11 +186,9 @@ export class KyselySecretService implements SecretService {
     const out: Record<string, unknown> = {}
     for (const row of rows) {
       try {
-        const kek = this.getKEK(row.keyVersion)
-        out[row.key] = await envelopeDecrypt(
-          kek,
-          row.ciphertext,
-          row.wrappedDek
+        const kek = await this.getKEK(row.keyVersion)
+        out[row.key] = createSecretValue(
+          await envelopeDecrypt(kek, row.ciphertext, row.wrappedDek)
         )
       } catch (cause) {
         // A stored secret that fails to decrypt is never expected — it means the
@@ -154,7 +204,7 @@ export class KyselySecretService implements SecretService {
         )
       }
     }
-    return out as T
+    return out as Partial<SecretValues<T>>
   }
 
   async rotateKEK(): Promise<number> {
@@ -168,12 +218,11 @@ export class KyselySecretService implements SecretService {
       .where('keyVersion', '<', this.keyVersion)
       .execute()
 
+    const oldKEK = await this.getKEK(this.keyVersion - 1)
+    const newKEK = await this.getKEK(this.keyVersion)
+
     for (const row of rows) {
-      const newWrappedDEK = await envelopeRewrap(
-        this.previousKey,
-        this.key,
-        row.wrappedDek
-      )
+      const newWrappedDEK = await envelopeRewrap(oldKEK, newKEK, row.wrappedDek)
       await this.db
         .updateTable('secrets')
         .set({

package/src/kysely-services.test.ts

@@ -1,4 +1,4 @@
-import { describe, test, before, after } from 'node:test'
+import { describe, test, before, beforeEach, after } from 'node:test'
 import assert from 'node:assert/strict'
 import { CamelCasePlugin, Kysely, SqliteDialect } from 'kysely'
 import { SerializePlugin } from './serialize-plugin.js'
@@ -54,8 +54,10 @@ async function dropAllTables(db: Kysely<KyselyPikkuDB>): Promise<void> {
     'workflow_runs',
     'workflow_versions',
     'secrets_audit',
+    'secret_kek_salts',
     'secrets',
     'credentials_audit',
+    'credential_kek_salts',
     'credentials',
   ]
   for (const table of tables) {
@@ -166,6 +168,126 @@ function registerTests(
     })
   })
 
+  describe(`KyselySecretService KEK derivation [${dialectName}]`, () => {
+    const kek = 'test-key-encryption-key-32chars!'
+
+    beforeEach(async () => {
+      await getDb().deleteFrom('secrets').execute()
+      await getDb().deleteFrom('secretKekSalts').execute()
+    })
+
+    test('a KEK salt row is created once per key version', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+      await service.setSecret('salt-a', 'one')
+      await service.setSecret('salt-b', 'two')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .select(['keyVersion', 'salt'])
+        .execute()
+
+      assert.equal(rows.length, 1)
+      assert.equal(rows[0]!.keyVersion, 1)
+      assert.ok(rows[0]!.salt.length > 0)
+    })
+
+    test('a second instance reuses the stored salt rather than minting one', async () => {
+      const first = new KyselySecretService(getDb(), { key: kek })
+      await first.init()
+      await first.setSecret('shared-salt', 'value')
+
+      const second = new KyselySecretService(getDb(), { key: kek })
+      await second.init()
+      assert.equal((await second.getSecret('shared-salt')).reveal(), 'value')
+
+      const rows = await getDb()
+        .selectFrom('secretKekSalts')
+        .selectAll()
+        .execute()
+      assert.equal(rows.length, 1)
+    })
+
+    test('getSecrets over 50 secrets costs one KEK derivation, not 50', async () => {
+      const service = new KyselySecretService(getDb(), { key: kek })
+      await service.init()
+
+      const keys: string[] = []
+      for (let i = 0; i < 50; i++) {
+        const key = `bulk-${i}`
+        keys.push(key)
+        await service.setSecret(key, `value-${i}`)
+      }
+
+      const reader = new KyselySecretService(getDb(), { key: kek })
+      await reader.init()
+
+      const start = performance.now()
+      const out = await reader.getSecrets(keys)
+      const elapsed = performance.now() - start
+
+      assert.equal(Object.keys(out).length, 50)
+      assert.equal(out['bulk-7']!.reveal(), 'value-7')
+      assert.ok(
+        elapsed < 400,
+        `getSecrets over 50 secrets took ${elapsed.toFixed(0)}ms; one derivation per secret would cost seconds`
+      )
+    })
+
+    test('rotateKEK over 50 secrets derives two KEKs, not 100', async () => {
+      const previousKey = 'previous-key-encryption-key-32c!'
+      const writer = new KyselySecretService(getDb(), {
+        key: previousKey,
+        keyVersion: 1,
+      })
+      await writer.init()
+
+      for (let i = 0; i < 50; i++) {
+        await writer.setSecret(`rotate-${i}`, `value-${i}`)
+      }
+
+      const rotator = new KyselySecretService(getDb(), {
+        key: kek,
+        keyVersion: 2,
+        previousKey,
+      })
+      await rotator.init()
+
+      const start = performance.now()
+      const rotated = await rotator.rotateKEK()
+      const elapsed = performance.now() - start
+
+      assert.equal(rotated, 50)
+      assert.equal((await rotator.getSecret('rotate-7')).reveal(), 'value-7')
+      assert.ok(
+        elapsed < 700,
+        `rotating 50 secrets took ${elapsed.toFixed(0)}ms; two derivations per secret would cost seconds`
+      )
+
+      const salts = await getDb()
+        .selectFrom('secretKekSalts')
+        .select('keyVersion')
+        .orderBy('keyVersion', 'asc')
+        .execute()
+      assert.deepEqual(
+        salts.map((r) => r.keyVersion),
+        [1, 2]
+      )
+    })
+
+    test('a wrong KEK passphrase fails loudly rather than returning garbage', async () => {
+      const writer = new KyselySecretService(getDb(), { key: kek })
+      await writer.init()
+      await writer.setSecret('wrong-kek', 'value')
+
+      const reader = new KyselySecretService(getDb(), {
+        key: 'a-completely-different-kek-32ch!',
+      })
+      await reader.init()
+      await assert.rejects(() => reader.getSecret('wrong-kek'))
+    })
+  })
+
   describe(`KyselyCredentialService audit [${dialectName}]`, () => {
     const kek = 'test-key-encryption-key-32chars!'
 

package/src/kysely-tables.test.ts

@@ -5,7 +5,15 @@ import { join } from 'node:path'
 import { Kysely, SqliteDialect } from 'kysely'
 import Database from 'better-sqlite3'
 
-import { applyPikkuSchemas } from './schema/index.js'
+import { applyPikkuSchemas, ensurePikkuSchema } from './schema/index.js'
+import { virtualUserSchema } from './schema/virtual-user.schema.js'
+
+/**
+ * Opt-in schemas that still have an interface in `KyselyPikkuDB`, so the drift
+ * check covers them too. A table being opt-in says when it is created, not
+ * whether its type is allowed to disagree with its DDL.
+ */
+const OPTIONAL_SCHEMAS = [virtualUserSchema]
 
 /**
  * `KyselyPikkuDB` and the schema declaration describe the same tables, and
@@ -65,6 +73,9 @@ const declaredSchema = async (): Promise<Map<string, Set<string>>> => {
       .addColumn('id', 'text', (col) => col.primaryKey())
       .execute()
     await applyPikkuSchemas(db)
+    for (const schema of OPTIONAL_SCHEMAS) {
+      await ensurePikkuSchema(db, schema)
+    }
 
     const tables = await db.introspection.getTables()
     return new Map(

package/src/kysely-tables.ts

@@ -131,7 +131,14 @@ export interface AIRunTable {
   agentName: string
   threadId: string
   resourceId: string
-  status: Generated<'running' | 'suspended' | 'completed' | 'failed'>
+  /**
+   * `interrupted` is distinct from `failed`: the run was stopped on purpose,
+   * usually because the user talked over it. Nothing went wrong, so it should
+   * not show up in error reporting.
+   */
+  status: Generated<
+    'running' | 'suspended' | 'completed' | 'failed' | 'interrupted'
+  >
   errorMessage: string | null
   suspendReason: 'approval' | 'credential' | 'rpc-missing' | null
   missingRpcs: string | null
@@ -165,6 +172,12 @@ export interface SecretsTable {
   updatedAt: Generated<Date>
 }
 
+export interface KekSaltsTable {
+  keyVersion: number
+  salt: string
+  createdAt: Generated<Date>
+}
+
 export interface SecretsAuditTable {
   id: string
   secretKey: string
@@ -215,10 +228,14 @@ export interface PikkuScopesTable {
   declared: Generated<boolean>
 }
 
-/** A role: an admin-composed bag of scopes. */
+/** A role: a bag of scopes, composed by an admin or declared in code. */
 export interface PikkuRolesTable {
   name: string
   description: string | null
+  /** Declared with `defineSystemRole`, and therefore not editable here. */
+  system: Generated<boolean>
+  /** False for a system role whose declaration has gone: held, but inert. */
+  declared: Generated<boolean>
   createdAt: Generated<Date>
 }
 
@@ -264,6 +281,37 @@ export interface WebhookDeliveryAttemptTable {
   createdAt: Generated<Date>
 }
 
+/**
+ * One virtual-user run. The JSON columns (`goals`, `memory`, `findings`,
+ * `tally`) are text the store serialises itself, so the row is byte-identical
+ * on every engine.
+ */
+export interface VirtualUserRunTable {
+  runId: string
+  persona: string
+  disposition: string
+  /**
+   * Read back as a string by some drivers — a BIGINT does not always fit a JS
+   * number — so the store narrows it rather than trusting the column type.
+   */
+  seed: string | number | bigint
+  status: Generated<'running' | 'completed' | 'failed'>
+  goals: Generated<string>
+  memory: Generated<string>
+  findings: Generated<string>
+  tally: string | null
+  stoppedBy: string | null
+  error: string | null
+  startedBy: string | null
+  /**
+   * Written as an ISO string and read back as whatever the driver returns — a
+   * bare SQLite driver cannot bind a `Date` at all, and the store normalises
+   * both ends.
+   */
+  createdAt: Generated<Date | string>
+  finishedAt: Date | string | null
+}
+
 export interface KyselyPikkuDB {
   pikkuScopes: PikkuScopesTable
   pikkuRoles: PikkuRolesTable
@@ -284,10 +332,13 @@ export interface KyselyPikkuDB {
   pikkuDeployments: PikkuDeploymentsTable
   pikkuDeploymentFunctions: PikkuDeploymentFunctionsTable
   secrets: SecretsTable
+  secretKekSalts: KekSaltsTable
   secretsAudit: SecretsAuditTable
   credentials: CredentialsTable
+  credentialKekSalts: KekSaltsTable
   credentialsAudit: CredentialsAuditTable
   pikkuUserSessions: UserSessionsTable
   webhookDelivery: WebhookDeliveryTable
   webhookDeliveryAttempt: WebhookDeliveryAttemptTable
+  virtualUserRun: VirtualUserRunTable
 }