@pikku/core 0.12.55 → 0.12.57

package/src/dev/hot-reload.ts

@@ -1,20 +1,20 @@
 import { watch, type FSWatcher } from 'node:fs'
-import { stat, readFile } from 'node:fs/promises'
-import { join, resolve, relative } from 'node:path'
+import { stat, readFile, copyFile, rm } from 'node:fs/promises'
+import { basename, dirname, join, resolve, relative } from 'node:path'
 import { pathToFileURL } from 'node:url'
 
 import { register } from 'tsx/esm/api'
 
 import { pikkuState } from '../pikku-state.js'
-import { addFunction } from '../function/function-runner.js'
 import { clearMiddlewareCache } from '../middleware-runner.js'
 import { clearPermissionsCache } from '../permissions.js'
 import { clearChannelMiddlewareCache } from '../wirings/channel/channel-middleware-runner.js'
 import { httpRouter } from '../wirings/http/routers/http-router.js'
-import { addSchema, compileAllSchemas } from '../schema.js'
 import type { Logger } from '../services/logger.js'
 import type { CorePikkuFunctionConfig } from '../function/functions.types.js'
 
+export * from './reload-meta.js'
+
 interface PikkuDevReloaderOptions {
   srcDirectories: string[]
   logger: Logger
@@ -65,16 +65,37 @@ const ensureTsxRegistered = () => {
   tsxRegistered = true
 }
 
+let tempCounter = 0
+
 const reimportModule = async (
   filePath: string,
   useTsx = false
 ): Promise<Record<string, unknown> | null> => {
   try {
     if (useTsx) {
-      ensureTsxRegistered()
-      return await import(
-        `${pathToFileURL(resolve(filePath)).href}?t=${Date.now()}`
+      // Import a uniquely-named sibling copy: a `?t=` query does NOT bust
+      // the cache on either runtime (Bun keys module identity on the bare
+      // path; tsx's transform cache keys on the file path too), so a fresh
+      // path is the only reliable re-import. Same directory → identical
+      // resolution for relative and package-`imports` (#…) specifiers. The
+      // dot-prefix keeps it out of the watcher.
+      if (!process.versions.bun) {
+        // Node needs tsx's loader to import raw .ts; Bun imports it natively.
+        ensureTsxRegistered()
+      }
+      const abs = resolve(filePath)
+      const tempPath = join(
+        dirname(abs),
+        `.pikku-hot-${++tempCounter}-${basename(abs)}`
       )
+      await copyFile(abs, tempPath)
+      try {
+        return await import(pathToFileURL(tempPath).href)
+      } finally {
+        await rm(tempPath, { force: true }).catch(() => {
+          // Best-effort temp cleanup; a leftover dotfile is watcher-ignored.
+        })
+      }
     }
 
     const content = await readFile(resolve(filePath), 'utf-8')
@@ -91,13 +112,23 @@ const isWatchedTsFile = (filename: string): boolean => {
     filename.endsWith('.ts') &&
     !filename.endsWith('.test.ts') &&
     !filename.endsWith('.d.ts') &&
-    !filename.endsWith('.gen.ts')
+    !filename.endsWith('.gen.ts') &&
+    // Hidden files: editor/sed atomic-write temps and our own hot-reload
+    // sibling copies must never trigger a reload of themselves.
+    !basename(filename).startsWith('.')
   )
 }
 
+export interface PikkuDevReloaderHandle {
+  close: () => void
+  /** Re-import every file changed since the last drain (post-codegen, once
+   *  fresh meta is in state). */
+  reimportPending: () => Promise<void>
+}
+
 export async function pikkuDevReloader(
   options: PikkuDevReloaderOptions
-): Promise<{ close: () => void }> {
+): Promise<PikkuDevReloaderHandle> {
   const { srcDirectories, logger, pikkuDir = '.pikku' } = options
   const absSrcDirs = srcDirectories.map((d) => resolve(d))
   const absPikkuDir = resolve(pikkuDir)
@@ -108,6 +139,7 @@ export async function pikkuDevReloader(
   const handleFileChange = async (changedTsFile: string) => {
     const start = Date.now()
     const reloadedNames: string[] = []
+    const addedNames: string[] = []
 
     const srcDir = absSrcDirs.find((d) => changedTsFile.startsWith(d))
     if (!srcDir) return
@@ -134,54 +166,59 @@ export async function pikkuDevReloader(
       return
     }
 
-    let schemasChanged = false
-
+    // Register every function-config export — replacing known functions AND
+    // adding new ones (a brand-new function becomes callable as soon as its
+    // meta lands via reloadGeneratedMeta after the next codegen pass).
+    // Write into the map captured at startup, NOT pikkuState's current one:
+    // a dev-server watcher may have temporarily swapped in a codegen-scoped
+    // map for the same file event (runAllWithCommandState), and a write to
+    // that map is silently discarded when it restores the original.
+    // Schemas are NOT touched here: `input`/`output` hold raw zod schemas,
+    // while the schema map carries codegen-generated JSON schemas — mixing the
+    // two crashed every reload. Fresh JSON schemas arrive via
+    // reloadGeneratedMeta once codegen has re-emitted them.
     for (const [exportName, exportValue] of Object.entries(mod)) {
-      if (isFunctionConfig(exportValue) && functionsMap.has(exportName)) {
-        addFunction(exportName, exportValue)
-        reloadedNames.push(exportName)
-
-        if (exportValue.input) {
-          addSchema(exportName, exportValue.input)
-          schemasChanged = true
-        }
-        if (exportValue.output) {
-          addSchema(`${exportName}Output`, exportValue.output)
-          schemasChanged = true
-        }
-      }
+      if (!isFunctionConfig(exportValue)) continue
+      const isNew = !functionsMap.has(exportName)
+      functionsMap.set(exportName, exportValue)
+      if (isNew) addedNames.push(exportName)
+      else reloadedNames.push(exportName)
     }
 
-    if (reloadedNames.length > 0) {
-      clearMiddlewareCache()
-      clearPermissionsCache()
-      clearChannelMiddlewareCache()
-      httpRouter.reset()
-
-      if (schemasChanged) {
-        try {
-          compileAllSchemas(logger)
-        } catch (err) {
-          const msg = err instanceof Error ? err.message : String(err)
-          if (
-            msg.includes('SchemaService') ||
-            (msg.includes('schema') && msg.includes('not'))
-          ) {
-            logger.warn('Schema recompilation skipped (no SchemaService)')
-          } else {
-            logger.error(`Schema recompilation failed: ${msg}`)
-            return
-          }
-        }
-      }
+    // Re-importing the module re-ran its wire* side effects (wireHTTP et al
+    // are keyed map-sets, so re-registration replaces/adds) — reset the
+    // router and caches even when no function export changed, so a
+    // wiring-only file edit rebuilds the route matchers too.
+    clearMiddlewareCache()
+    clearPermissionsCache()
+    clearChannelMiddlewareCache()
+    httpRouter.reset()
 
+    if (reloadedNames.length > 0 || addedNames.length > 0) {
       const elapsed = Date.now() - start
-      logger.info(`Hot-reloaded: ${reloadedNames.join(', ')} (${elapsed}ms)`)
+      const parts: string[] = []
+      if (reloadedNames.length > 0) parts.push(reloadedNames.join(', '))
+      if (addedNames.length > 0) parts.push(`new: ${addedNames.join(', ')}`)
+      logger.info(`Hot-reloaded: ${parts.join('; ')} (${elapsed}ms)`)
     }
   }
 
   let debounceTimer: ReturnType<typeof setTimeout> | undefined
   const pendingChanges = new Set<string>()
+  // Files re-imported since the last reimportPending() drain. A dev-server
+  // watcher drains this after its codegen pass so wire* registrations that
+  // were skipped for missing meta (a NEW route) run again with fresh meta.
+  const postCodegenQueue = new Set<string>()
+
+  const safeHandleFileChange = async (file: string) => {
+    try {
+      await handleFileChange(file)
+    } catch (err) {
+      logger.error(
+        `Hot-reload error for ${relative(process.cwd(), file)}: ${err instanceof Error ? err.message : String(err)}`
+      )
+    }
+  }
 
   const scheduleReload = (filePath: string) => {
     pendingChanges.add(filePath)
@@ -190,13 +227,8 @@ export async function pikkuDevReloader(
       const files = [...pendingChanges]
       pendingChanges.clear()
       for (const file of files) {
-        try {
-          await handleFileChange(file)
-        } catch (err) {
-          logger.error(
-            `Hot-reload error for ${relative(process.cwd(), file)}: ${err instanceof Error ? err.message : String(err)}`
-          )
-        }
+        postCodegenQueue.add(file)
+        await safeHandleFileChange(file)
       }
     }, 50)
   }
@@ -229,5 +261,18 @@ export async function pikkuDevReloader(
         watcher.close()
       }
     },
+    // Drain the queue of recently re-imported files and import them again.
+    // A dev-server watcher calls this AFTER its codegen pass has refreshed
+    // the generated meta (reloadGeneratedMeta): wire* registrations skip
+    // routes whose meta doesn't exist yet, so a wiring file changed
+    // alongside a NEW function only registers its new route when
+    // re-imported after the fresh meta has landed.
+    reimportPending: async () => {
+      const files = [...postCodegenQueue]
+      postCodegenQueue.clear()
+      for (const file of files) {
+        await safeHandleFileChange(file)
+      }
+    },
   }
 }

package/src/dev/reload-meta.test.ts

@@ -0,0 +1,154 @@
+import { describe, test, beforeEach, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtemp, writeFile, rm, mkdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+
+import { pikkuState, resetPikkuState } from '../pikku-state.js'
+import { getSchema } from '../schema.js'
+import { reloadGeneratedMeta } from './reload-meta.js'
+
+const createMockLogger = () => {
+  const logs: Array<{ level: string; message: string }> = []
+  return {
+    info: (msg: string) => logs.push({ level: 'info', message: String(msg) }),
+    warn: (msg: string) => logs.push({ level: 'warn', message: String(msg) }),
+    error: (msg: string | Error) =>
+      logs.push({
+        level: 'error',
+        message: msg instanceof Error ? msg.message : String(msg),
+      }),
+    debug: (msg: string) => logs.push({ level: 'debug', message: String(msg) }),
+    getLogs: () => logs,
+    setLevel: () => {},
+  }
+}
+
+const compiled: string[] = []
+const mockSchemaService = {
+  compileSchema: (name: string, _value: any) => {
+    compiled.push(name)
+  },
+  validateSchema: () => {},
+  getSchemaNames: () => new Set(compiled),
+  getSchemaKeys: () => [],
+} as any
+
+describe('reloadGeneratedMeta', { concurrency: false }, () => {
+  let tmpDir: string
+  let mockLogger: ReturnType<typeof createMockLogger>
+
+  beforeEach(async () => {
+    resetPikkuState()
+    compiled.length = 0
+    tmpDir = await mkdtemp(join(tmpdir(), 'pikku-reload-meta-test-'))
+    mockLogger = createMockLogger()
+  })
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true })
+  })
+
+  test('reads generated meta json into pikku state', async () => {
+    await mkdir(join(tmpDir, 'function'), { recursive: true })
+    await mkdir(join(tmpDir, 'rpc'), { recursive: true })
+    await writeFile(
+      join(tmpDir, 'function/pikku-functions-meta.gen.json'),
+      JSON.stringify({
+        newFunc: { pikkuFuncId: 'newFunc', inputSchemaName: 'NewFuncInput' },
+      })
+    )
+    await writeFile(
+      join(tmpDir, 'rpc/pikku-rpc-wirings-meta.internal.gen.json'),
+      JSON.stringify({ newFunc: 'newFunc' })
+    )
+
+    await reloadGeneratedMeta({
+      pikkuDir: tmpDir,
+      logger: mockLogger,
+      schemaService: mockSchemaService,
+    })
+
+    const functionsMeta = pikkuState(null, 'function', 'meta') as any
+    assert.equal(functionsMeta.newFunc.pikkuFuncId, 'newFunc')
+    const rpcMeta = pikkuState(null, 'rpc', 'meta') as any
+    assert.equal(rpcMeta.newFunc, 'newFunc')
+  })
+
+  test('preserves runtime-registered internals (workflow orchestrator) across reload', async () => {
+    // The workflow service registers these at init; they are never in the
+    // generated JSON, so the reload must merge — not replace — the meta maps.
+    pikkuState(null, 'function', 'meta', {
+      pikkuWorkflowOrchestrator: { pikkuFuncId: 'pikkuWorkflowOrchestrator' },
+    })
+    pikkuState(null, 'queue', 'meta', {
+      'pikku-workflow-orchestrator': {
+        pikkuFuncId: 'pikkuWorkflowOrchestrator',
+        name: 'pikku-workflow-orchestrator',
+      },
+    })
+
+    await mkdir(join(tmpDir, 'function'), { recursive: true })
+    await mkdir(join(tmpDir, 'queue'), { recursive: true })
+    await writeFile(
+      join(tmpDir, 'function/pikku-functions-meta.gen.json'),
+      JSON.stringify({ userFunc: { pikkuFuncId: 'userFunc' } })
+    )
+    await writeFile(
+      join(tmpDir, 'queue/pikku-queue-workers-wirings-meta.gen.json'),
+      JSON.stringify({ userQueue: { pikkuFuncId: 'userFunc', name: 'userQueue' } })
+    )
+
+    await reloadGeneratedMeta({
+      pikkuDir: tmpDir,
+      logger: mockLogger,
+      schemaService: mockSchemaService,
+    })
+
+    const functionsMeta = pikkuState(null, 'function', 'meta') as any
+    assert.equal(
+      functionsMeta.pikkuWorkflowOrchestrator?.pikkuFuncId,
+      'pikkuWorkflowOrchestrator',
+      'runtime-registered internal must survive the reload'
+    )
+    assert.equal(functionsMeta.userFunc?.pikkuFuncId, 'userFunc')
+    const queueMeta = pikkuState(null, 'queue', 'meta') as any
+    assert.equal(
+      queueMeta['pikku-workflow-orchestrator']?.pikkuFuncId,
+      'pikkuWorkflowOrchestrator'
+    )
+    assert.equal(queueMeta.userQueue?.name, 'userQueue')
+  })
+
+  test('re-adds generated json schemas and recompiles them', async () => {
+    const schemasDir = join(tmpDir, 'schemas', 'schemas')
+    await mkdir(schemasDir, { recursive: true })
+    await writeFile(
+      join(schemasDir, 'NewFuncInput.schema.json'),
+      JSON.stringify({ type: 'object', properties: { id: { type: 'string' } } })
+    )
+
+    await reloadGeneratedMeta({
+      pikkuDir: tmpDir,
+      logger: mockLogger,
+      schemaService: mockSchemaService,
+    })
+
+    const schema = getSchema('NewFuncInput') as any
+    assert.equal(schema.type, 'object')
+    assert.ok(
+      compiled.includes('NewFuncInput'),
+      'Schema should be recompiled after reload'
+    )
+  })
+
+  test('missing meta files and schemas dir are not an error', async () => {
+    await reloadGeneratedMeta({
+      pikkuDir: tmpDir,
+      logger: mockLogger,
+      schemaService: mockSchemaService,
+    })
+    const errors = mockLogger.getLogs().filter((l) => l.level === 'error')
+    assert.deepEqual(errors, [])
+  })
+})

package/src/dev/reload-meta.ts

@@ -0,0 +1,138 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+
+import { pikkuState } from '../pikku-state.js'
+import { addSchema, compileAllSchemas } from '../schema.js'
+import { clearMiddlewareCache } from '../middleware-runner.js'
+import { clearPermissionsCache } from '../permissions.js'
+import { clearChannelMiddlewareCache } from '../wirings/channel/channel-middleware-runner.js'
+import { httpRouter } from '../wirings/http/routers/http-router.js'
+import type { Logger } from '../services/logger.js'
+import type { SchemaService } from '../services/schema-service.js'
+
+export interface ReloadGeneratedMetaOptions {
+  /** The project's generated output directory (the CLI's resolved outDir). */
+  pikkuDir: string
+  logger: Logger
+  /** Used to recompile validators for changed schemas; falls back to the
+   *  schema service on the registered singleton services. */
+  schemaService?: SchemaService
+}
+
+const readJson = async (
+  logger: Logger,
+  file: string
+): Promise<any | undefined> => {
+  let raw: string
+  try {
+    raw = await readFile(file, 'utf-8')
+  } catch {
+    // The project doesn't use this wiring type — nothing generated.
+    return undefined
+  }
+  try {
+    return JSON.parse(raw)
+  } catch (err) {
+    logger.error(
+      `Hot-reload could not parse ${file}: ${err instanceof Error ? err.message : String(err)}`
+    )
+    return undefined
+  }
+}
+
+/**
+ * Re-reads the codegen output (wiring meta + JSON schemas) into the running
+ * process so new and changed functions become callable without a server
+ * restart. The generated `*-meta.gen.ts` files are plain
+ * `pikkuState(area, key, <json>)` side effects, but they cannot be
+ * re-imported after a dev-time codegen — the ESM cache pins both the wrapper
+ * and its JSON import — so this reads the JSON sources directly and applies
+ * the same state.
+ *
+ * Meant to be called by a dev-server watcher after each codegen pass. Routes
+ * registered by NEW `wireHTTP` files are not picked up (their modules were
+ * never imported); those still need a restart.
+ */
+export async function reloadGeneratedMeta(
+  options: ReloadGeneratedMetaOptions
+): Promise<void> {
+  const { pikkuDir, logger, schemaService } = options
+  const dir = resolve(pikkuDir)
+
+  const functionsMeta = await readJson(
+    logger,
+    join(dir, 'function/pikku-functions-meta.gen.json')
+  )
+  // Merge over the existing map, don't replace it: framework internals like
+  // pikkuWorkflowOrchestrator / the per-workflow queue workers are registered
+  // at service-init (pikku-workflow-service.ts), never in the generated JSON —
+  // a wholesale replace drops them and workflow jobs then fail with
+  // "Function meta not found: pikkuWorkflowOrchestrator".
+  if (functionsMeta) {
+    const existing = pikkuState(null, 'function', 'meta') ?? {}
+    pikkuState(null, 'function', 'meta', { ...existing, ...functionsMeta })
+  }
+
+  const httpMeta = await readJson(
+    logger,
+    join(dir, 'http/pikku-http-wirings-meta.gen.json')
+  )
+  if (httpMeta) pikkuState(null, 'http', 'meta', httpMeta)
+
+  const rpcMeta = await readJson(
+    logger,
+    join(dir, 'rpc/pikku-rpc-wirings-meta.internal.gen.json')
+  )
+  if (rpcMeta) pikkuState(null, 'rpc', 'meta', rpcMeta)
+
+  const queueMeta = await readJson(
+    logger,
+    join(dir, 'queue/pikku-queue-workers-wirings-meta.gen.json')
+  )
+  // Same reason as function meta: the workflow service adds its orchestrator /
+  // step queues (and wf-orchestrator-* / wf-step-* per-workflow queues) here at
+  // init, absent from the generated JSON — merge so they survive the reload.
+  if (queueMeta) {
+    const existing = pikkuState(null, 'queue', 'meta') ?? {}
+    pikkuState(null, 'queue', 'meta', { ...existing, ...queueMeta })
+  }
+
+  const agentMeta = await readJson(
+    logger,
+    join(dir, 'agent/pikku-agent-wirings-meta.gen.json')
+  )
+  if (agentMeta?.agentsMeta) {
+    pikkuState(null, 'agent', 'agentsMeta', agentMeta.agentsMeta)
+  }
+
+  // Generated JSON schemas: <outDir>/schemas/schemas/<Name>.schema.json.
+  // Re-adding replaces the map entry; the schema service recompiles any
+  // validator whose stored schema value no longer matches.
+  const schemasDir = join(dir, 'schemas', 'schemas')
+  let schemaFiles: string[] = []
+  try {
+    schemaFiles = await readdir(schemasDir)
+  } catch {
+    // No generated schemas — a schema-less project.
+  }
+  for (const file of schemaFiles) {
+    if (!file.endsWith('.schema.json')) continue
+    const schema = await readJson(logger, join(schemasDir, file))
+    if (schema) {
+      addSchema(file.slice(0, -'.schema.json'.length), schema)
+    }
+  }
+
+  clearMiddlewareCache()
+  clearPermissionsCache()
+  clearChannelMiddlewareCache()
+  httpRouter.reset()
+
+  try {
+    compileAllSchemas(logger, schemaService)
+  } catch (err) {
+    logger.error(
+      `Hot-reload schema recompilation failed: ${err instanceof Error ? err.message : String(err)}`
+    )
+  }
+}

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -921,14 +921,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   ): JobOptions {
     const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES
     const retryDelay = stepOptions?.retryDelay
+    // A concrete retryDelay (15000, '15s') is a fixed backoff; only the literal
+    // 'exponential' — or no delay at all — selects exponential.
     const backoff =
-      typeof retryDelay === 'number'
-        ? { type: 'fixed', delay: retryDelay }
-        : retryDelay === 'exponential'
+      retryDelay !== undefined && retryDelay !== 'exponential'
+        ? { type: 'fixed', delay: getDurationInMilliseconds(retryDelay) }
+        : retries > 0 || retryDelay === 'exponential'
           ? 'exponential'
-          : retries > 0
-            ? 'exponential'
-            : undefined
+          : undefined
     return { attempts: retries + 1, ...(backoff ? { backoff } : {}) }
   }
 

package/src/wirings/workflow/workflow-retry-policy.test.ts

@@ -48,6 +48,13 @@ describe('resolveStepJobOptions — workflow owns retry policy', () => {
     })
   })
 
+  test("duration-string retryDelay ('15s') → fixed backoff in ms", () => {
+    assert.deepEqual(ws.resolve({ retries: 3, retryDelay: '15s' }), {
+      attempts: 4,
+      backoff: { type: 'fixed', delay: 15000 },
+    })
+  })
+
   test("retryDelay: 'exponential' → exponential backoff", () => {
     assert.deepEqual(ws.resolve({ retries: 2, retryDelay: 'exponential' }), {
       attempts: 3,