@pikku/redis 0.12.12 → 0.12.14

package/src/redis-secret-service.ts

@@ -1,6 +1,8 @@
 import type { SecretService } from '@pikku/core/services'
 import { Redis, type RedisOptions } from 'ioredis'
 import {
+  deriveKEK,
+  generateKEKSalt,
   envelopeEncrypt,
   envelopeDecrypt,
   envelopeRewrap,
@@ -20,6 +22,8 @@ export class RedisSecretService implements SecretService {
   private keyVersion: number
   private previousKey?: string
   private keyPrefix: string
+  private kekSalts = new Map<number, string>()
+  private keks = new Map<number, CryptoKey>()
 
   constructor(
     connectionOrConfig: Redis | RedisOptions | string,
@@ -50,17 +54,49 @@ export class RedisSecretService implements SecretService {
     return `${this.keyPrefix}:secret:${key}`
   }
 
-  private getKEK(version: number): string {
-    if (version === this.keyVersion) return this.key
-    if (this.previousKey) return this.previousKey
-    throw new Error(`No KEK available for key_version ${version}`)
+  private saltKey(): string {
+    return `${this.keyPrefix}:kek-salt`
+  }
+
+  private async getKEKSalt(version: number): Promise<string> {
+    const cached = this.kekSalts.get(version)
+    if (cached) return cached
+
+    const field = String(version)
+    let salt = await this.redis.hget(this.saltKey(), field)
+    if (!salt) {
+      await this.redis.hsetnx(this.saltKey(), field, generateKEKSalt())
+      salt = await this.redis.hget(this.saltKey(), field)
+      if (!salt) {
+        throw new Error(
+          `Failed to persist a KEK salt for key_version ${version}`
+        )
+      }
+    }
+
+    this.kekSalts.set(version, salt)
+    return salt
+  }
+
+  private async getKEK(version: number): Promise<CryptoKey> {
+    const cached = this.keks.get(version)
+    if (cached) return cached
+
+    const passphrase = version === this.keyVersion ? this.key : this.previousKey
+    if (!passphrase) {
+      throw new Error(`No KEK available for key_version ${version}`)
+    }
+
+    const kek = await deriveKEK(passphrase, await this.getKEKSalt(version))
+    this.keks.set(version, kek)
+    return kek
   }
 
   async getSecret<T = string>(key: string): Promise<T> {
     const data = await this.redis.hgetall(this.secretKey(key))
     if (!data.ciphertext) throw new Error('Requested secret not found')
 
-    const kek = this.getKEK(Number(data.key_version))
+    const kek = await this.getKEK(Number(data.key_version))
     return envelopeDecrypt<T>(kek, data.ciphertext!, data.wrapped_dek!)
   }
 
@@ -70,7 +106,10 @@ export class RedisSecretService implements SecretService {
   }
 
   async setSecret(key: string, value: unknown): Promise<void> {
-    const { ciphertext, wrappedDEK } = await envelopeEncrypt(this.key, value)
+    const { ciphertext, wrappedDEK } = await envelopeEncrypt(
+      await this.getKEK(this.keyVersion),
+      value
+    )
 
     await this.redis.hset(this.secretKey(key), {
       ciphertext,
@@ -83,15 +122,32 @@ export class RedisSecretService implements SecretService {
     await this.redis.del(this.secretKey(key))
   }
 
-  async getSecrets<
-    T extends Record<string, unknown> = Record<string, unknown>,
-  >(keys: (keyof T & string)[]): Promise<T> {
-    const results = await Promise.allSettled(keys.map((k) => this.getSecret(k)))
+  async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
+    keys: (keyof T & string)[]
+  ): Promise<T> {
+    const rows = await Promise.all(
+      keys.map(async (key) => ({
+        key,
+        data: await this.redis.hgetall(this.secretKey(key)),
+      }))
+    )
+
     const out: Record<string, unknown> = {}
-    keys.forEach((key, i) => {
-      const result = results[i]
-      if (result?.status === 'fulfilled') out[key] = result.value
-    })
+    for (const { key, data } of rows) {
+      if (!data.ciphertext) continue
+
+      const keyVersion = Number(data.key_version)
+      try {
+        const kek = await this.getKEK(keyVersion)
+        out[key] = await envelopeDecrypt(kek, data.ciphertext, data.wrapped_dek!)
+      } catch (cause) {
+        throw new Error(
+          `Failed to decrypt secret "${key}" (key_version ${keyVersion}): ` +
+            `the configured KEK does not match the key it was wrapped under`,
+          { cause }
+        )
+      }
+    }
     return out as T
   }
 
@@ -100,6 +156,9 @@ export class RedisSecretService implements SecretService {
       throw new Error('No previousKey configured — nothing to rotate from')
     }
 
+    const oldKEK = await this.getKEK(this.keyVersion - 1)
+    const newKEK = await this.getKEK(this.keyVersion)
+
     const pattern = `${this.keyPrefix}:secret:*`
     let cursor = '0'
     let count = 0
@@ -120,8 +179,8 @@ export class RedisSecretService implements SecretService {
         if (version >= this.keyVersion) continue
 
         const newWrappedDEK = await envelopeRewrap(
-          this.previousKey,
-          this.key,
+          oldKEK,
+          newKEK,
           data.wrapped_dek!
         )
         await this.redis.hset(redisKey, {

package/src/redis-workflow-run-service.ts

@@ -167,48 +167,6 @@ export class RedisWorkflowRunService implements WorkflowRunService {
     }
   }
 
-  async getAIGeneratedWorkflows(
-    agentName?: string
-  ): Promise<Array<{ workflowName: string; graphHash: string; graph: any }>> {
-    const pattern = agentName
-      ? `${this.keyPrefix}:version:ai:${agentName}:*`
-      : `${this.keyPrefix}:version:ai:*`
-    const results: Array<{
-      workflowName: string
-      graphHash: string
-      graph: any
-    }> = []
-    let cursor = '0'
-    do {
-      const [nextCursor, keys] = await this.redis.scan(
-        cursor,
-        'MATCH',
-        pattern,
-        'COUNT',
-        100
-      )
-      cursor = nextCursor
-      for (const key of keys) {
-        const data = await this.redis.hgetall(key)
-        if (
-          data.source === 'ai-agent' &&
-          data.graph &&
-          data.status === 'active'
-        ) {
-          const workflowName = data.workflowName
-          const graphHash = data.graphHash
-          if (!workflowName || !graphHash) continue
-          results.push({
-            workflowName,
-            graphHash,
-            graph: JSON.parse(data.graph),
-          })
-        }
-      }
-    } while (cursor !== '0')
-    return results
-  }
-
   async deleteRun(id: string): Promise<boolean> {
     const exists = await this.redis.exists(this.runKey(id))
     if (!exists) return false

package/src/redis-workflow-service.test.ts

@@ -0,0 +1,115 @@
+import { describe, test, beforeEach, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+import Redis from 'ioredis-mock'
+
+import { RedisWorkflowService } from './redis-workflow-service.js'
+
+const KEY_PREFIX = 'workflows'
+
+let ws: RedisWorkflowService
+let redis: any
+
+const newRun = () =>
+  ws.createRun('flow', {}, false, 'hash', { type: 'test' } as any)
+
+beforeEach(() => {
+  redis = new Redis()
+  ws = new RedisWorkflowService(redis, KEY_PREFIX)
+})
+
+afterEach(() => {
+  redis.disconnect()
+})
+
+describe('workflow run state in redis', () => {
+  test('two branches setting different keys both survive', async () => {
+    const runId = await newRun()
+
+    await Promise.all([
+      ws.updateRunState(runId, 'left', 'a'),
+      ws.updateRunState(runId, 'right', 'b'),
+    ])
+
+    assert.deepEqual(
+      await ws.getRunState(runId),
+      { left: 'a', right: 'b' },
+      'a parallel branch overwrote the other branch’s state'
+    )
+  })
+
+  test('many concurrent writers all land', async () => {
+    const runId = await newRun()
+    const keys = Array.from({ length: 20 }, (_, i) => `k${i}`)
+
+    await Promise.all(keys.map((k, i) => ws.updateRunState(runId, k, i)))
+
+    const state = await ws.getRunState(runId)
+    assert.equal(Object.keys(state).length, keys.length)
+    keys.forEach((k, i) => assert.equal(state[k], i))
+  })
+
+  test('an empty array stays an array when another key is written', async () => {
+    const runId = await newRun()
+
+    await ws.updateRunState(runId, 'items', [])
+    await ws.updateRunState(runId, 'count', 0)
+
+    const state = await ws.getRunState(runId)
+    assert.ok(
+      Array.isArray(state.items),
+      `an empty array came back as ${JSON.stringify(state.items)}`
+    )
+    assert.equal(state.count, 0)
+  })
+
+  test('values keep their types, including null and nested objects', async () => {
+    const runId = await newRun()
+
+    await ws.updateRunState(runId, 'nothing', null)
+    await ws.updateRunState(runId, 'nested', { a: [1, { b: true }] })
+    await ws.updateRunState(runId, 'flag', false)
+
+    assert.deepEqual(await ws.getRunState(runId), {
+      nothing: null,
+      nested: { a: [1, { b: true }] },
+      flag: false,
+    })
+  })
+
+  test('a later write to the same key replaces it', async () => {
+    const runId = await newRun()
+
+    await ws.updateRunState(runId, 'attempt', 1)
+    await ws.updateRunState(runId, 'attempt', 2)
+
+    assert.deepEqual(await ws.getRunState(runId), { attempt: 2 })
+  })
+
+  test('the state of a run that has none is empty', async () => {
+    assert.deepEqual(await ws.getRunState(await newRun()), {})
+  })
+
+  test('state written before the per-key layout is still readable', async () => {
+    const runId = await newRun()
+    const redis = (ws as any).redis
+    await redis.hset(
+      (ws as any).runKey(runId),
+      'state',
+      JSON.stringify({ legacy: 'kept', shared: 'old' })
+    )
+
+    assert.deepEqual(await ws.getRunState(runId), {
+      legacy: 'kept',
+      shared: 'old',
+    })
+
+    await ws.updateRunState(runId, 'shared', 'new')
+    await ws.updateRunState(runId, 'fresh', 1)
+
+    assert.deepEqual(
+      await ws.getRunState(runId),
+      { legacy: 'kept', shared: 'new', fresh: 1 },
+      'a run in flight across the deploy lost the state it already had'
+    )
+  })
+})

package/src/redis-workflow-service.ts

@@ -42,9 +42,17 @@ export class RedisWorkflowService extends PikkuWorkflowService {
     super(options)
     this.keyPrefix = keyPrefix
 
-    // Check if it's a Redis instance or config options
-    if (connectionOrConfig instanceof Redis) {
-      this.redis = connectionOrConfig
+    // An already-connected client is recognised by the commands it exposes
+    // rather than by `instanceof Redis`, so a compatible client that is not an
+    // ioredis subclass — a test double, a wrapper — is used as given instead of
+    // being silently ignored in favour of a fresh connection to localhost.
+    if (
+      typeof connectionOrConfig === 'object' &&
+      connectionOrConfig !== null &&
+      'hgetall' in connectionOrConfig &&
+      'hset' in connectionOrConfig
+    ) {
+      this.redis = connectionOrConfig as Redis
       this.ownsConnection = false
     } else {
       this.redis = new Redis(connectionOrConfig as any)
@@ -821,26 +829,9 @@ export class RedisWorkflowService extends PikkuWorkflowService {
     return { completedNodeIds, failedNodeIds, branchKeys }
   }
 
-  async getNodesWithoutSteps(
-    runId: string,
-    nodeIds: string[]
-  ): Promise<string[]> {
-    if (nodeIds.length === 0) return []
-
-    const result: string[] = []
-
-    for (const nodeId of nodeIds) {
-      const key = this.stepKey(runId, nodeId)
-      const exists = await this.redis.exists(key)
-      if (!exists) {
-        result.push(nodeId)
-      }
-    }
-
-    return result
-  }
-
-  async getStepInstances(runId: string): Promise<
+  async getStepInstances(
+    runId: string
+  ): Promise<
     Array<{ stepName: string; status: StepStatus; fromStepName?: string }>
   > {
     const instances: Array<{
@@ -920,37 +911,50 @@ export class RedisWorkflowService extends PikkuWorkflowService {
     )
   }
 
+  /**
+   * Hash holding one field per state variable of a run.
+   *
+   * The whole state used to be a single JSON blob on the run hash, which made
+   * every write a read-modify-write: two parallel branches each setting their
+   * own variable would both read the blob, add their key, and write it back,
+   * so whichever wrote second silently dropped the other's variable.
+   */
+  private runStateKey(runId: string): string {
+    return `${this.keyPrefix}:run-state:${runId}`
+  }
+
   protected async updateRunStateImpl(
     runId: string,
     name: string,
     value: unknown
   ): Promise<void> {
-    const key = this.runKey(runId)
-    const now = Date.now()
-
-    // Get existing state or create empty object
-    const existingState = await this.redis.hget(key, 'state')
-    const state = existingState ? JSON.parse(existingState) : {}
-
-    // Update the specific field
-    state[name] = value
-
-    await this.redis.hmset(
-      key,
-      'state',
-      JSON.stringify(state),
+    // HSET touches only the field it is named, so concurrent writers to
+    // different variables no longer race, and nothing has to parse and
+    // re-encode the values it is not writing.
+    await this.redis.hset(
+      this.runStateKey(runId),
+      name,
+      JSON.stringify(value) ?? 'null'
+    )
+    await this.redis.hset(
+      this.runKey(runId),
       'updatedAt',
-      now.toString()
+      Date.now().toString()
     )
   }
 
   async getRunState(runId: string): Promise<Record<string, unknown>> {
-    const key = this.runKey(runId)
-    const stateStr = await this.redis.hget(key, 'state')
-    if (!stateStr) {
-      return {}
+    // The blob is what a run started before the per-key layout still holds; a
+    // field written since wins, so a run in flight across the deploy keeps
+    // everything it had.
+    const blob = await this.redis.hget(this.runKey(runId), 'state')
+    const state: Record<string, unknown> = blob ? JSON.parse(blob) : {}
+
+    const fields = await this.redis.hgetall(this.runStateKey(runId))
+    for (const [name, value] of Object.entries(fields)) {
+      state[name] = JSON.parse(value)
     }
-    return JSON.parse(stateStr)
+    return state
   }
 
   private versionKey(name: string, graphHash: string): string {
@@ -1007,48 +1011,6 @@ export class RedisWorkflowService extends PikkuWorkflowService {
     }
   }
 
-  async getAIGeneratedWorkflows(
-    agentName?: string
-  ): Promise<Array<{ workflowName: string; graphHash: string; graph: any }>> {
-    const pattern = agentName
-      ? `${this.keyPrefix}:version:ai\\:${agentName}\\:*`
-      : `${this.keyPrefix}:version:ai\\:*`
-    const results: Array<{
-      workflowName: string
-      graphHash: string
-      graph: any
-    }> = []
-    let cursor = '0'
-    do {
-      const [nextCursor, keys] = await this.redis.scan(
-        cursor,
-        'MATCH',
-        pattern,
-        'COUNT',
-        100
-      )
-      cursor = nextCursor
-      for (const key of keys) {
-        const data = await this.redis.hgetall(key)
-        if (
-          !data.graph ||
-          data.source !== 'ai-agent' ||
-          data.status !== 'active'
-        )
-          continue
-        const workflowName = data.workflowName
-        const graphHash = data.graphHash
-        if (!workflowName || !graphHash) continue
-        results.push({
-          workflowName,
-          graphHash,
-          graph: JSON.parse(data.graph),
-        })
-      }
-    } while (cursor !== '0')
-    return results
-  }
-
   async close(): Promise<void> {
     if (this.ownsConnection) {
       await this.redis.quit()