@pikku/core 0.12.71 → 0.12.74

package/src/crypto-utils.ts

@@ -1,3 +1,5 @@
+import { WeakKeyMaterialError } from './errors/errors.js'
+
 const encoder = new TextEncoder()
 const decoder = new TextDecoder()
 
@@ -45,13 +47,35 @@ const fromBase64Url = (input: string): Uint8Array => {
   return base64Decode(base64)
 }
 
-const deriveKey = async (secret: string): Promise<CryptoKey> => {
+const SALT_BYTES = 16
+const IV_BYTES = 12
+const TAG_BYTES = 16
+const PBKDF2_ITERATIONS = 600_000
+
+const deriveKey = async (
+  secret: string,
+  salt: Uint8Array
+): Promise<CryptoKey> => {
   const subtle = getSubtle()
-  const hash = await subtle.digest('SHA-256', encoder.encode(secret))
-  return subtle.importKey('raw', hash, { name: 'AES-GCM' }, false, [
-    'encrypt',
-    'decrypt',
-  ])
+  const material = await subtle.importKey(
+    'raw',
+    encoder.encode(secret),
+    { name: 'PBKDF2' },
+    false,
+    ['deriveKey']
+  )
+  return subtle.deriveKey(
+    {
+      name: 'PBKDF2',
+      salt: salt as BufferSource,
+      iterations: PBKDF2_ITERATIONS,
+      hash: 'SHA-256',
+    },
+    material,
+    { name: 'AES-GCM', length: 256 },
+    false,
+    ['encrypt', 'decrypt']
+  )
 }
 
 export const encryptJSON = async (
@@ -63,8 +87,9 @@ export const encryptJSON = async (
     throw new Error('WebCrypto not available')
   }
   const subtle = getSubtle()
-  const iv = crypto.getRandomValues(new Uint8Array(12))
-  const key = await deriveKey(secret)
+  const salt = crypto.getRandomValues(new Uint8Array(SALT_BYTES))
+  const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES))
+  const key = await deriveKey(secret, salt)
   const plaintext = encoder.encode(JSON.stringify(value))
   const encrypted = await subtle.encrypt(
     { name: 'AES-GCM', iv },
@@ -72,9 +97,10 @@ export const encryptJSON = async (
     plaintext
   )
   const cipherBytes = new Uint8Array(encrypted)
-  const out = new Uint8Array(iv.length + cipherBytes.length)
-  out.set(iv, 0)
-  out.set(cipherBytes, iv.length)
+  const out = new Uint8Array(salt.length + iv.length + cipherBytes.length)
+  out.set(salt, 0)
+  out.set(iv, salt.length)
+  out.set(cipherBytes, salt.length + iv.length)
   return toBase64Url(out)
 }
 
@@ -84,12 +110,13 @@ export const decryptJSON = async <T>(
 ): Promise<T> => {
   const subtle = getSubtle()
   const data = fromBase64Url(token)
-  if (data.length < 13) {
+  if (data.length <= SALT_BYTES + IV_BYTES + TAG_BYTES) {
     throw new Error('Invalid encrypted payload')
   }
-  const iv = data.slice(0, 12)
-  const ciphertext = data.slice(12)
-  const key = await deriveKey(secret)
+  const salt = data.slice(0, SALT_BYTES)
+  const iv = data.slice(SALT_BYTES, SALT_BYTES + IV_BYTES)
+  const ciphertext = data.slice(SALT_BYTES + IV_BYTES)
+  const key = await deriveKey(secret, salt)
   const decrypted = await subtle.decrypt(
     { name: 'AES-GCM', iv },
     key,
@@ -99,58 +126,202 @@ export const decryptJSON = async <T>(
 }
 
 /**
- * Envelope encryption utilities.
- *
- * Each secret gets its own random DEK (data encryption key).
- * The DEK is wrapped (encrypted) by a KEK (key encryption key) — typically an env var.
- * The actual secret is encrypted with the DEK.
- *
- * KEK rotation only re-wraps the DEK, never touches the ciphertext.
+ * Namespaces the remote-RPC session key so the same deployment secret used for
+ * another purpose derives a different key. See knowledge/crypto.md.
  */
+export const REMOTE_SESSION_INFO = 'pikku:remote-session'
 
-const importRawKey = async (rawBytes: Uint8Array): Promise<CryptoKey> => {
+export const MIN_KEY_MATERIAL_LENGTH = 32
+
+export const assertStrongKeyMaterial = (
+  name: string,
+  keyMaterial: string
+): void => {
+  if (keyMaterial.length < MIN_KEY_MATERIAL_LENGTH) {
+    throw new WeakKeyMaterialError(
+      name,
+      MIN_KEY_MATERIAL_LENGTH,
+      keyMaterial.length
+    )
+  }
+}
+
+const expandKeyMaterial = async (
+  keyMaterial: string,
+  info: string,
+  salt: Uint8Array
+): Promise<CryptoKey> => {
   const subtle = getSubtle()
-  return subtle.importKey(
+  const material = await subtle.importKey(
     'raw',
-    rawBytes.buffer as ArrayBuffer,
-    { name: 'AES-GCM' },
-    true,
+    encoder.encode(keyMaterial),
+    'HKDF',
+    false,
+    ['deriveKey']
+  )
+  return subtle.deriveKey(
+    {
+      name: 'HKDF',
+      hash: 'SHA-256',
+      salt: salt as BufferSource,
+      info: encoder.encode(info) as BufferSource,
+    },
+    material,
+    { name: 'AES-GCM', length: 256 },
+    false,
     ['encrypt', 'decrypt']
   )
 }
 
-export const generateDEK = async (): Promise<string> => {
-  const raw = globalThis.crypto.getRandomValues(new Uint8Array(32))
-  return toBase64Url(raw)
+export const encryptWithKeyMaterial = async (
+  name: string,
+  keyMaterial: string,
+  info: string,
+  value: unknown
+): Promise<string> => {
+  assertStrongKeyMaterial(name, keyMaterial)
+  const crypto = globalThis.crypto
+  if (!crypto?.getRandomValues) {
+    throw new Error('WebCrypto not available')
+  }
+  const subtle = getSubtle()
+  const salt = crypto.getRandomValues(new Uint8Array(SALT_BYTES))
+  const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES))
+  const key = await expandKeyMaterial(keyMaterial, info, salt)
+  const encrypted = await subtle.encrypt(
+    { name: 'AES-GCM', iv },
+    key,
+    encoder.encode(JSON.stringify(value))
+  )
+  const cipherBytes = new Uint8Array(encrypted)
+  const out = new Uint8Array(salt.length + iv.length + cipherBytes.length)
+  out.set(salt, 0)
+  out.set(iv, salt.length)
+  out.set(cipherBytes, salt.length + iv.length)
+  return toBase64Url(out)
 }
 
-export const wrapDEK = async (
-  kek: string,
-  plaintextDEK: string
-): Promise<string> => {
-  return encryptJSON(kek, plaintextDEK)
+export const decryptWithKeyMaterial = async <T>(
+  name: string,
+  keyMaterial: string,
+  info: string,
+  token: string
+): Promise<T> => {
+  assertStrongKeyMaterial(name, keyMaterial)
+  const subtle = getSubtle()
+  const data = fromBase64Url(token)
+  if (data.length <= SALT_BYTES + IV_BYTES + TAG_BYTES) {
+    throw new Error('Invalid encrypted payload')
+  }
+  const salt = data.slice(0, SALT_BYTES)
+  const iv = data.slice(SALT_BYTES, SALT_BYTES + IV_BYTES)
+  const ciphertext = data.slice(SALT_BYTES + IV_BYTES)
+  const key = await expandKeyMaterial(keyMaterial, info, salt)
+  const decrypted = await subtle.decrypt(
+    { name: 'AES-GCM', iv },
+    key,
+    ciphertext
+  )
+  return JSON.parse(decoder.decode(new Uint8Array(decrypted))) as T
 }
 
-export const unwrapDEK = async (
-  kek: string,
-  wrappedDEK: string
+export const encodeBase64UrlText = (value: string): string =>
+  toBase64Url(encoder.encode(value))
+
+export const decodeBase64UrlText = (value: string): string =>
+  decoder.decode(fromBase64Url(value))
+
+const expandSigningKeyMaterial = async (
+  keyMaterial: string,
+  info: string
+): Promise<CryptoKey> => {
+  const subtle = getSubtle()
+  const material = await subtle.importKey(
+    'raw',
+    encoder.encode(keyMaterial),
+    'HKDF',
+    false,
+    ['deriveKey']
+  )
+  return subtle.deriveKey(
+    {
+      name: 'HKDF',
+      hash: 'SHA-256',
+      salt: new Uint8Array(0) as BufferSource,
+      info: encoder.encode(info) as BufferSource,
+    },
+    material,
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign', 'verify']
+  )
+}
+
+/**
+ * HMAC-SHA256 over `payload` under a key expanded from high-entropy key
+ * material, base64url-encoded. The salt is empty by design so the verifier can
+ * re-derive the key from `keyMaterial` and `info` alone.
+ */
+export const signWithKeyMaterial = async (
+  name: string,
+  keyMaterial: string,
+  info: string,
+  payload: string
 ): Promise<string> => {
-  return decryptJSON<string>(kek, wrappedDEK)
+  assertStrongKeyMaterial(name, keyMaterial)
+  const key = await expandSigningKeyMaterial(keyMaterial, info)
+  const signature = await getSubtle().sign(
+    'HMAC',
+    key,
+    encoder.encode(payload) as BufferSource
+  )
+  return toBase64Url(new Uint8Array(signature))
 }
 
-const encryptWithDEK = async (
-  dekBase64: string,
+/** False — never throws — for a malformed, truncated or mismatched signature. */
+export const verifyWithKeyMaterial = async (
+  name: string,
+  keyMaterial: string,
+  info: string,
+  payload: string,
+  signature: string
+): Promise<boolean> => {
+  assertStrongKeyMaterial(name, keyMaterial)
+  const key = await expandSigningKeyMaterial(keyMaterial, info)
+  try {
+    return await getSubtle().verify(
+      'HMAC',
+      key,
+      fromBase64Url(signature) as BufferSource,
+      encoder.encode(payload) as BufferSource
+    )
+  } catch {
+    return false
+  }
+}
+
+const importRawKey = async (rawBytes: Uint8Array): Promise<CryptoKey> => {
+  const subtle = getSubtle()
+  return subtle.importKey(
+    'raw',
+    rawBytes.buffer as ArrayBuffer,
+    { name: 'AES-GCM' },
+    true,
+    ['encrypt', 'decrypt']
+  )
+}
+
+const encryptWithCryptoKey = async (
+  key: CryptoKey,
   value: unknown
 ): Promise<string> => {
   const crypto = globalThis.crypto
   const subtle = getSubtle()
-  const iv = crypto.getRandomValues(new Uint8Array(12))
-  const key = await importRawKey(fromBase64Url(dekBase64))
-  const plaintext = encoder.encode(JSON.stringify(value))
+  const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES))
   const encrypted = await subtle.encrypt(
     { name: 'AES-GCM', iv },
     key,
-    plaintext
+    encoder.encode(JSON.stringify(value))
   )
   const cipherBytes = new Uint8Array(encrypted)
   const out = new Uint8Array(iv.length + cipherBytes.length)
@@ -159,18 +330,17 @@ const encryptWithDEK = async (
   return toBase64Url(out)
 }
 
-const decryptWithDEK = async <T>(
-  dekBase64: string,
+const decryptWithCryptoKey = async <T>(
+  key: CryptoKey,
   token: string
 ): Promise<T> => {
   const subtle = getSubtle()
   const data = fromBase64Url(token)
-  if (data.length < 13) {
+  if (data.length < IV_BYTES + TAG_BYTES) {
     throw new Error('Invalid encrypted payload')
   }
-  const iv = data.slice(0, 12)
-  const ciphertext = data.slice(12)
-  const key = await importRawKey(fromBase64Url(dekBase64))
+  const iv = data.slice(0, IV_BYTES)
+  const ciphertext = data.slice(IV_BYTES)
   const decrypted = await subtle.decrypt(
     { name: 'AES-GCM', iv },
     key,
@@ -179,13 +349,71 @@ const decryptWithDEK = async <T>(
   return JSON.parse(decoder.decode(new Uint8Array(decrypted))) as T
 }
 
+export const generateDEK = async (): Promise<string> => {
+  const raw = globalThis.crypto.getRandomValues(new Uint8Array(32))
+  return toBase64Url(raw)
+}
+
+/**
+ * One salt per (deployment, key version), stored beside the wrapped DEKs.
+ * See knowledge/crypto.md.
+ */
+export const generateKEKSalt = (): string => {
+  const crypto = globalThis.crypto
+  if (!crypto?.getRandomValues) {
+    throw new Error('WebCrypto not available')
+  }
+  return toBase64Url(crypto.getRandomValues(new Uint8Array(SALT_BYTES)))
+}
+
+export const deriveKEK = async (
+  passphrase: string,
+  salt: string
+): Promise<CryptoKey> => {
+  return deriveKey(passphrase, fromBase64Url(salt))
+}
+
+export const wrapDEK = async (
+  kek: CryptoKey,
+  plaintextDEK: string
+): Promise<string> => {
+  return encryptWithCryptoKey(kek, plaintextDEK)
+}
+
+export const unwrapDEK = async (
+  kek: CryptoKey,
+  wrappedDEK: string
+): Promise<string> => {
+  return decryptWithCryptoKey<string>(kek, wrappedDEK)
+}
+
+const encryptWithDEK = async (
+  dekBase64: string,
+  value: unknown
+): Promise<string> => {
+  return encryptWithCryptoKey(
+    await importRawKey(fromBase64Url(dekBase64)),
+    value
+  )
+}
+
+const decryptWithDEK = async <T>(
+  dekBase64: string,
+  token: string
+): Promise<T> => {
+  return decryptWithCryptoKey<T>(
+    await importRawKey(fromBase64Url(dekBase64)),
+    token
+  )
+}
+
 export interface EnvelopeEncryptResult {
   ciphertext: string
   wrappedDEK: string
 }
 
 export const envelopeEncrypt = async (
-  kek: string,
+  kek: CryptoKey,
   value: unknown
 ): Promise<EnvelopeEncryptResult> => {
   const dek = await generateDEK()
@@ -195,7 +423,7 @@ export const envelopeEncrypt = async (
 }
 
 export const envelopeDecrypt = async <T>(
-  kek: string,
+  kek: CryptoKey,
   ciphertext: string,
   wrappedDEK: string
 ): Promise<T> => {
@@ -204,8 +432,8 @@ export const envelopeDecrypt = async <T>(
 }
 
 export const envelopeRewrap = async (
-  oldKEK: string,
-  newKEK: string,
+  oldKEK: CryptoKey,
+  newKEK: CryptoKey,
   wrappedDEK: string
 ): Promise<string> => {
   const dek = await unwrapDEK(oldKEK, wrappedDEK)

package/src/data-classification.ts

@@ -1,10 +1,4 @@
-// The `__classification__` marker is OPTIONAL on purpose. A required property
-// would make a plain value (e.g. `string`) unassignable to a branded column
-// (`Private<string>`), which breaks ordinary Kysely query operands —
-// `where('email', '=', someString)`, inserts, and `.set(...)`. Making it
-// optional keeps the brand structurally present (so the inspector's PKU910
-// output check still detects it) while allowing plain values to flow IN.
-// See `findPiiPaths` in @pikku/inspector, which reads the level union-aware.
+// knowledge: decisions/internals/core-data-classification-brand-is-an-optional-property.md
 export type Private<T> = T & { readonly __classification__?: 'private' }
 export type Pii<T> = T & { readonly __classification__?: 'pii' }
 export type Secret<T> = T & { readonly __classification__?: 'secret' }

package/src/dev/hot-reload.test.ts

@@ -354,14 +354,12 @@ describe('pikkuDevReloader', { concurrency: false }, () => {
       pikkuDir: tmpDir,
     })
 
-    // Write updated function via file system and let the watcher pick it up
     const jsV2 = `export const hotTask = { func: async () => { return { reloaded: true }; }, auth: false };\n`
     await writeFile(join(tmpDir, 'hotTask.js'), jsV2)
     await writeFile(join(tmpDir, 'hotTask.ts'), `// trigger ${Date.now()}`)
 
     await wait(300)
 
-    // Verify the function was reloaded via the watcher
     const reloadLog = mockLogger
       .getLogs()
       .find(
@@ -370,8 +368,6 @@ describe('pikkuDevReloader', { concurrency: false }, () => {
       )
     assert.ok(reloadLog, 'Should log hot-reload for hotTask')
 
-    // runScheduledTask uses runPikkuFunc which reads from the functions Map,
-    // so it will pick up the hot-reloaded function (no longer sets taskResult.ref)
     await runScheduledTask({ name: 'hotTask' })
 
     // The reloaded function no longer sets taskResult.ref, so it should

package/src/dev/hot-reload.ts

@@ -52,12 +52,6 @@ const findCompiledFile = async (
   return null
 }
 
-// Changed user files are re-run through an evictable module runner keyed by a
-// stable path (createModuleRunner), NOT re-imported under a fresh URL. A
-// fresh-URL import (a `data:` URL on Node, a uniquely-named temp sibling on
-// Bun) permanently leaks a record in the native ESM loader map on every
-// reload, OOMing long editing sessions; the runner overwrites a single slot so
-// the previous module is collected. See ./module-runner.ts.
 const isWatchedTsFile = (filename: string): boolean => {
   return (
     filename.endsWith('.ts') &&
@@ -110,17 +104,12 @@ export async function pikkuDevReloader(
       return
     }
 
-    // Register every function-config export — replacing known functions AND
-    // adding new ones (a brand-new function becomes callable as soon as its
-    // meta lands via reloadGeneratedMeta after the next codegen pass).
-    // Write into the map captured at startup, NOT pikkuState's current one:
-    // a dev-server watcher may have temporarily swapped in a codegen-scoped
-    // map for the same file event (runAllWithCommandState), and a write to
-    // that map is silently discarded when it restores the original.
-    // Schemas are NOT touched here: `input`/`output` hold raw zod schemas,
-    // while the schema map carries codegen-generated JSON schemas — mixing the
-    // two crashed every reload. Fresh JSON schemas arrive via
-    // reloadGeneratedMeta once codegen has re-emitted them.
+    // Register every function-config export, replacing known functions and
+    // adding new ones. Write into the map captured at startup, NOT pikkuState's
+    // current one: a dev-server watcher may have temporarily swapped in a
+    // codegen-scoped map, whose writes are discarded when it restores.
+    // Schemas are NOT touched here: `input`/`output` hold raw zod schemas while
+    // the schema map carries codegen JSON schemas; mixing them crashed reloads.
     for (const [exportName, exportValue] of Object.entries(mod)) {
       if (!isFunctionConfig(exportValue)) continue
       const isNew = !functionsMap.has(exportName)
@@ -129,10 +118,9 @@ export async function pikkuDevReloader(
       else reloadedNames.push(exportName)
     }
 
-    // Re-importing the module re-ran its wire* side effects (wireHTTP et al
-    // are keyed map-sets, so re-registration replaces/adds) — reset the
-    // router and caches even when no function export changed, so a
-    // wiring-only file edit rebuilds the route matchers too.
+    // Re-importing re-ran the module’s wire* side effects, so reset the router
+    // and caches even when no function export changed — a wiring-only edit has
+    // to rebuild the route matchers too.
     clearMiddlewareCache()
     clearPermissionsCache()
     clearChannelMiddlewareCache()
@@ -149,9 +137,8 @@ export async function pikkuDevReloader(
 
   let debounceTimer: ReturnType<typeof setTimeout> | undefined
   const pendingChanges = new Set<string>()
-  // Files re-imported since the last reimportPending() drain. A dev-server
-  // watcher drains this after its codegen pass so wire* registrations that
-  // were skipped for missing meta (a NEW route) run again with fresh meta.
+  // Files re-imported since the last reimportPending() drain, re-run after the
+  // dev server's codegen pass so registrations skipped for missing meta retry.
   const postCodegenQueue = new Set<string>()
 
   const safeHandleFileChange = async (file: string) => {
@@ -206,12 +193,6 @@ export async function pikkuDevReloader(
       }
       moduleRunner.clear()
     },
-    // Drain the queue of recently re-imported files and import them again.
-    // A dev-server watcher calls this AFTER its codegen pass has refreshed
-    // the generated meta (reloadGeneratedMeta): wire* registrations skip
-    // routes whose meta doesn't exist yet, so a wiring file changed
-    // alongside a NEW function only registers its new route when
-    // re-imported after the fresh meta has landed.
     reimportPending: async () => {
       const files = [...postCodegenQueue]
       postCodegenQueue.clear()

package/src/dev/module-runner.ts

@@ -5,46 +5,26 @@ import { pathToFileURL } from 'node:url'
 import { compileFunction } from 'node:vm'
 import type { transformSync as EsbuildTransformSync } from 'esbuild'
 
-// Evictable module runner for hot-reload.
-//
-// Re-importing a changed user file through the native ESM loader (a fresh
-// `data:` URL on Node, a fresh temp-file path on Bun) is unbounded: the loader
-// keeps a `Map<url, moduleRecord>` for the realm lifetime with no eviction API,
-// so every reload permanently leaks a module record and the dev server
-// eventually OOMs (worse on Bun). Instead we own the registry: transpile the
-// source to CJS (esbuild), run it as a plain function via `vm.compileFunction`
-// — which produces a garbage-collectable function rather than a permanent
-// loader entry — and store the resulting exports under a STABLE path key. A
-// reload overwrites that one slot, so the previous module becomes unreachable
-// and is collected. `import`s inside the user file are delegated to
-// `createRequire`, whose resolution matches the native loader AND returns the
-// same live singletons (Node/Bun share the require/import module cache), so
-// top-level side effects (wireHTTP et al) mutate the same running services.
 type EsbuildTransform = typeof EsbuildTransformSync
 
 let transformSync: EsbuildTransform | undefined
 
 const loadTransform = async (): Promise<EsbuildTransform> => {
   if (transformSync) return transformSync
-  // esbuild is a dev-only dependency, hoisted from @pikku/cli at runtime — the
-  // same lazy pattern the reloader previously used for tsx. It is never loaded
-  // in production runtimes (the `./dev` export is dev-only).
+  // esbuild is a dev-only dependency hoisted from @pikku/cli at runtime; the
+  // `./dev` export is dev-only so it is never loaded in production runtimes.
   const esbuild = await import('esbuild')
   transformSync = esbuild.transformSync
   return transformSync
 }
 
 export interface PikkuModuleRunner {
-  /** Run a user module by absolute path, returning its exports. Repeated runs
-   *  of the same path overwrite a single registry slot (the previous module is
-   *  collected). Returns `null` if transpile/compile/evaluation fails, so the
-   *  caller can keep the previously-loaded code. */
+  /** Run a user module by absolute path. Repeated runs of one path overwrite a
+   *  single registry slot. Returns `null` on failure so the caller can keep the
+   *  previously-loaded code. */
   run: (absPath: string) => Promise<Record<string, unknown> | null>
-  /** Drop a single module from the registry. */
   evict: (absPath: string) => void
-  /** Drop every module from the registry. */
   clear: () => void
-  /** Number of modules currently held. */
   readonly size: number
 }
 
@@ -58,10 +38,6 @@ export const createModuleRunner = (): PikkuModuleRunner => {
     try {
       const transform = await loadTransform()
       const source = await readFile(absPath, 'utf-8')
-      // No sourcemap: an inline sourcemap embeds a base64 copy of the source
-      // that the engine retains per compile, reintroducing linear growth on
-      // Node — the exact leak this runner exists to remove. `filename` still
-      // anchors stack traces to the user file.
       const { code } = transform(source, {
         loader: absPath.endsWith('.ts') ? 'ts' : 'js',
         format: 'cjs',
@@ -81,9 +57,8 @@ export const createModuleRunner = (): PikkuModuleRunner => {
       registry.set(absPath, moduleObj.exports)
       return moduleObj.exports
     } catch {
-      // Transpile/compile/evaluate failure (a bad edit, or the one known
-      // limitation — a file using top-level `await`, which can't be emitted in
-      // `cjs` form). Return null so the caller keeps the previously-loaded code.
+      // A bad edit, or the one known limitation: a file using top-level
+      // `await`, which cannot be emitted in `cjs` form.
       return null
     }
   }