@pikku/core 0.12.55 → 0.12.57

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.12.57
+
+### Patch Changes
+
+- 60ad8cb: fix dev-server hot reload so edited AND new functions/routes apply without a restart
+  - `@pikku/core`: the hot reloader fed raw zod `input`/`output` schemas into the JSON-schema map, so `compileAllSchemas` threw `Failed to compile schema` on every reload and the reload aborted (only the function body sometimes swapped, half-updated). It now registers function implementations only and leaves schemas to the codegen JSON output. New function exports are registered too (previously only already-registered names were replaced). Reloads write into the startup functions map directly to avoid a race with the dev watcher's codegen-scoped state swap, and re-import via a uniquely-named sibling copy since neither Bun nor tsx bust the module cache on a `?t=` query.
+  - New `reloadGeneratedMeta` (exported from `@pikku/core/dev`) re-reads the regenerated wiring meta + JSON schemas into the running process so new/changed routes, RPCs, queues and agents resolve without a restart.
+  - `@pikku/cli`: `pikku dev` now calls `reloadGeneratedMeta` after each watch-triggered codegen pass and re-imports the changed files once fresh meta is in state, so a NEW route in a changed wiring file registers (its `wireHTTP` no longer no-ops on missing meta).
+  - `@pikku/schema-cfworker`: `compileSchema` recompiles when a schema's value changes (not only on first sight), so hot-reloaded schemas take effect.
+
+- 8f5c998: Fix dev hot-reload dropping runtime-registered function/queue meta. `reloadGeneratedMeta` replaced the whole `function`/`queue` meta maps with the generated JSON, wiping entries the framework registers at service-init (the workflow orchestrator, per-workflow queue workers, and other `addFunction`'d internals that never appear in the generated files). Workflow jobs then failed with `Function meta not found: pikkuWorkflowOrchestrator`. The reload now merges over the existing maps so those internals survive.
+
+## 0.12.56
+
+### Patch Changes
+
+- 6c30861: fix workflow step retry backoff firing immediately
+  - `@pikku/queue-pg-boss`: `backoff: 'exponential'` mapped to `retryBackoff: true` without a base `retryDelay`; pg-boss computes exponential backoff as `retry_delay * 2^n` with a queue default of 0, so every retry fired immediately. Exponential backoff now gets a 1s base delay, and sub-second fixed delays round up to 1s instead of flooring to 0 (= immediate).
+  - `@pikku/core`: a duration-string `retryDelay` (e.g. `'15s'`) on a workflow step was silently dropped (only numbers were honored) and fell back to exponential. It now resolves to a fixed backoff via `getDurationInMilliseconds`.
+
 ## 0.12.55
 
 ### Patch Changes

package/dist/dev/hot-reload.d.ts

@@ -1,10 +1,14 @@
 import type { Logger } from '../services/logger.js';
+export * from './reload-meta.js';
 interface PikkuDevReloaderOptions {
     srcDirectories: string[];
     logger: Logger;
     pikkuDir?: string;
 }
-export declare function pikkuDevReloader(options: PikkuDevReloaderOptions): Promise<{
+export interface PikkuDevReloaderHandle {
     close: () => void;
-}>;
-export {};
+    /** Re-import every file changed since the last drain (post-codegen, once
+     *  fresh meta is in state). */
+    reimportPending: () => Promise<void>;
+}
+export declare function pikkuDevReloader(options: PikkuDevReloaderOptions): Promise<PikkuDevReloaderHandle>;

package/dist/dev/hot-reload.js

@@ -1,15 +1,14 @@
 import { watch } from 'node:fs';
-import { stat, readFile } from 'node:fs/promises';
-import { join, resolve, relative } from 'node:path';
+import { stat, readFile, copyFile, rm } from 'node:fs/promises';
+import { basename, dirname, join, resolve, relative } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { register } from 'tsx/esm/api';
 import { pikkuState } from '../pikku-state.js';
-import { addFunction } from '../function/function-runner.js';
 import { clearMiddlewareCache } from '../middleware-runner.js';
 import { clearPermissionsCache } from '../permissions.js';
 import { clearChannelMiddlewareCache } from '../wirings/channel/channel-middleware-runner.js';
 import { httpRouter } from '../wirings/http/routers/http-router.js';
-import { addSchema, compileAllSchemas } from '../schema.js';
+export * from './reload-meta.js';
 const isFunctionConfig = (value) => {
     return (typeof value === 'object' &&
         value !== null &&
@@ -44,11 +43,31 @@ const ensureTsxRegistered = () => {
     register();
     tsxRegistered = true;
 };
+let tempCounter = 0;
 const reimportModule = async (filePath, useTsx = false) => {
     try {
         if (useTsx) {
-            ensureTsxRegistered();
-            return await import(`${pathToFileURL(resolve(filePath)).href}?t=${Date.now()}`);
+            // Import a uniquely-named sibling copy: a `?t=` query does NOT bust
+            // the cache on either runtime (Bun keys module identity on the bare
+            // path; tsx's transform cache keys on the file path too), so a fresh
+            // path is the only reliable re-import. Same directory → identical
+            // resolution for relative and package-`imports` (#…) specifiers. The
+            // dot-prefix keeps it out of the watcher.
+            if (!process.versions.bun) {
+                // Node needs tsx's loader to import raw .ts; Bun imports it natively.
+                ensureTsxRegistered();
+            }
+            const abs = resolve(filePath);
+            const tempPath = join(dirname(abs), `.pikku-hot-${++tempCounter}-${basename(abs)}`);
+            await copyFile(abs, tempPath);
+            try {
+                return await import(pathToFileURL(tempPath).href);
+            }
+            finally {
+                await rm(tempPath, { force: true }).catch(() => {
+                    // Best-effort temp cleanup; a leftover dotfile is watcher-ignored.
+                });
+            }
         }
         const content = await readFile(resolve(filePath), 'utf-8');
         const dataUrl = 'data:text/javascript;base64,' + Buffer.from(content).toString('base64');
@@ -62,7 +81,10 @@ const isWatchedTsFile = (filename) => {
     return (filename.endsWith('.ts') &&
         !filename.endsWith('.test.ts') &&
         !filename.endsWith('.d.ts') &&
-        !filename.endsWith('.gen.ts'));
+        !filename.endsWith('.gen.ts') &&
+        // Hidden files: editor/sed atomic-write temps and our own hot-reload
+        // sibling copies must never trigger a reload of themselves.
+        !basename(filename).startsWith('.'));
 };
 export async function pikkuDevReloader(options) {
     const { srcDirectories, logger, pikkuDir = '.pikku' } = options;
@@ -73,6 +95,7 @@ export async function pikkuDevReloader(options) {
     const handleFileChange = async (changedTsFile) => {
         const start = Date.now();
         const reloadedNames = [];
+        const addedNames = [];
         const srcDir = absSrcDirs.find((d) => changedTsFile.startsWith(d));
         if (!srcDir)
             return;
@@ -87,48 +110,59 @@ export async function pikkuDevReloader(options) {
             logger.error(`Failed to import: ${relative(process.cwd(), importPath)} (keeping old code)`);
             return;
         }
-        let schemasChanged = false;
+        // Register every function-config export — replacing known functions AND
+        // adding new ones (a brand-new function becomes callable as soon as its
+        // meta lands via reloadGeneratedMeta after the next codegen pass).
+        // Write into the map captured at startup, NOT pikkuState's current one:
+        // a dev-server watcher may have temporarily swapped in a codegen-scoped
+        // map for the same file event (runAllWithCommandState), and a write to
+        // that map is silently discarded when it restores the original.
+        // Schemas are NOT touched here: `input`/`output` hold raw zod schemas,
+        // while the schema map carries codegen-generated JSON schemas — mixing the
+        // two crashed every reload. Fresh JSON schemas arrive via
+        // reloadGeneratedMeta once codegen has re-emitted them.
         for (const [exportName, exportValue] of Object.entries(mod)) {
-            if (isFunctionConfig(exportValue) && functionsMap.has(exportName)) {
-                addFunction(exportName, exportValue);
+            if (!isFunctionConfig(exportValue))
+                continue;
+            const isNew = !functionsMap.has(exportName);
+            functionsMap.set(exportName, exportValue);
+            if (isNew)
+                addedNames.push(exportName);
+            else
                 reloadedNames.push(exportName);
-                if (exportValue.input) {
-                    addSchema(exportName, exportValue.input);
-                    schemasChanged = true;
-                }
-                if (exportValue.output) {
-                    addSchema(`${exportName}Output`, exportValue.output);
-                    schemasChanged = true;
-                }
-            }
         }
-        if (reloadedNames.length > 0) {
-            clearMiddlewareCache();
-            clearPermissionsCache();
-            clearChannelMiddlewareCache();
-            httpRouter.reset();
-            if (schemasChanged) {
-                try {
-                    compileAllSchemas(logger);
-                }
-                catch (err) {
-                    const msg = err instanceof Error ? err.message : String(err);
-                    if (msg.includes('SchemaService') ||
-                        (msg.includes('schema') && msg.includes('not'))) {
-                        logger.warn('Schema recompilation skipped (no SchemaService)');
-                    }
-                    else {
-                        logger.error(`Schema recompilation failed: ${msg}`);
-                        return;
-                    }
-                }
-            }
+        // Re-importing the module re-ran its wire* side effects (wireHTTP et al
+        // are keyed map-sets, so re-registration replaces/adds) — reset the
+        // router and caches even when no function export changed, so a
+        // wiring-only file edit rebuilds the route matchers too.
+        clearMiddlewareCache();
+        clearPermissionsCache();
+        clearChannelMiddlewareCache();
+        httpRouter.reset();
+        if (reloadedNames.length > 0 || addedNames.length > 0) {
             const elapsed = Date.now() - start;
-            logger.info(`Hot-reloaded: ${reloadedNames.join(', ')} (${elapsed}ms)`);
+            const parts = [];
+            if (reloadedNames.length > 0)
+                parts.push(reloadedNames.join(', '));
+            if (addedNames.length > 0)
+                parts.push(`new: ${addedNames.join(', ')}`);
+            logger.info(`Hot-reloaded: ${parts.join('; ')} (${elapsed}ms)`);
         }
     };
     let debounceTimer;
     const pendingChanges = new Set();
+    // Files re-imported since the last reimportPending() drain. A dev-server
+    // watcher drains this after its codegen pass so wire* registrations that
+    // were skipped for missing meta (a NEW route) run again with fresh meta.
+    const postCodegenQueue = new Set();
+    const safeHandleFileChange = async (file) => {
+        try {
+            await handleFileChange(file);
+        }
+        catch (err) {
+            logger.error(`Hot-reload error for ${relative(process.cwd(), file)}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    };
     const scheduleReload = (filePath) => {
         pendingChanges.add(filePath);
         if (debounceTimer)
@@ -137,12 +171,8 @@ export async function pikkuDevReloader(options) {
             const files = [...pendingChanges];
             pendingChanges.clear();
             for (const file of files) {
-                try {
-                    await handleFileChange(file);
-                }
-                catch (err) {
-                    logger.error(`Hot-reload error for ${relative(process.cwd(), file)}: ${err instanceof Error ? err.message : String(err)}`);
-                }
+                postCodegenQueue.add(file);
+                await safeHandleFileChange(file);
             }
         }, 50);
     };
@@ -168,5 +198,18 @@ export async function pikkuDevReloader(options) {
                 watcher.close();
             }
         },
+        // Drain the queue of recently re-imported files and import them again.
+        // A dev-server watcher calls this AFTER its codegen pass has refreshed
+        // the generated meta (reloadGeneratedMeta): wire* registrations skip
+        // routes whose meta doesn't exist yet, so a wiring file changed
+        // alongside a NEW function only registers its new route when
+        // re-imported after the fresh meta has landed.
+        reimportPending: async () => {
+            const files = [...postCodegenQueue];
+            postCodegenQueue.clear();
+            for (const file of files) {
+                await safeHandleFileChange(file);
+            }
+        },
     };
 }

package/dist/dev/reload-meta.d.ts

@@ -0,0 +1,24 @@
+import type { Logger } from '../services/logger.js';
+import type { SchemaService } from '../services/schema-service.js';
+export interface ReloadGeneratedMetaOptions {
+    /** The project's generated output directory (the CLI's resolved outDir). */
+    pikkuDir: string;
+    logger: Logger;
+    /** Used to recompile validators for changed schemas; falls back to the
+     *  schema service on the registered singleton services. */
+    schemaService?: SchemaService;
+}
+/**
+ * Re-reads the codegen output (wiring meta + JSON schemas) into the running
+ * process so new and changed functions become callable without a server
+ * restart. The generated `*-meta.gen.ts` files are plain
+ * `pikkuState(area, key, <json>)` side effects, but they cannot be
+ * re-imported after a dev-time codegen — the ESM cache pins both the wrapper
+ * and its JSON import — so this reads the JSON sources directly and applies
+ * the same state.
+ *
+ * Meant to be called by a dev-server watcher after each codegen pass. Routes
+ * registered by NEW `wireHTTP` files are not picked up (their modules were
+ * never imported); those still need a restart.
+ */
+export declare function reloadGeneratedMeta(options: ReloadGeneratedMetaOptions): Promise<void>;

package/dist/dev/reload-meta.js

@@ -0,0 +1,99 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { pikkuState } from '../pikku-state.js';
+import { addSchema, compileAllSchemas } from '../schema.js';
+import { clearMiddlewareCache } from '../middleware-runner.js';
+import { clearPermissionsCache } from '../permissions.js';
+import { clearChannelMiddlewareCache } from '../wirings/channel/channel-middleware-runner.js';
+import { httpRouter } from '../wirings/http/routers/http-router.js';
+const readJson = async (logger, file) => {
+    let raw;
+    try {
+        raw = await readFile(file, 'utf-8');
+    }
+    catch {
+        // The project doesn't use this wiring type — nothing generated.
+        return undefined;
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch (err) {
+        logger.error(`Hot-reload could not parse ${file}: ${err instanceof Error ? err.message : String(err)}`);
+        return undefined;
+    }
+};
+/**
+ * Re-reads the codegen output (wiring meta + JSON schemas) into the running
+ * process so new and changed functions become callable without a server
+ * restart. The generated `*-meta.gen.ts` files are plain
+ * `pikkuState(area, key, <json>)` side effects, but they cannot be
+ * re-imported after a dev-time codegen — the ESM cache pins both the wrapper
+ * and its JSON import — so this reads the JSON sources directly and applies
+ * the same state.
+ *
+ * Meant to be called by a dev-server watcher after each codegen pass. Routes
+ * registered by NEW `wireHTTP` files are not picked up (their modules were
+ * never imported); those still need a restart.
+ */
+export async function reloadGeneratedMeta(options) {
+    const { pikkuDir, logger, schemaService } = options;
+    const dir = resolve(pikkuDir);
+    const functionsMeta = await readJson(logger, join(dir, 'function/pikku-functions-meta.gen.json'));
+    // Merge over the existing map, don't replace it: framework internals like
+    // pikkuWorkflowOrchestrator / the per-workflow queue workers are registered
+    // at service-init (pikku-workflow-service.ts), never in the generated JSON —
+    // a wholesale replace drops them and workflow jobs then fail with
+    // "Function meta not found: pikkuWorkflowOrchestrator".
+    if (functionsMeta) {
+        const existing = pikkuState(null, 'function', 'meta') ?? {};
+        pikkuState(null, 'function', 'meta', { ...existing, ...functionsMeta });
+    }
+    const httpMeta = await readJson(logger, join(dir, 'http/pikku-http-wirings-meta.gen.json'));
+    if (httpMeta)
+        pikkuState(null, 'http', 'meta', httpMeta);
+    const rpcMeta = await readJson(logger, join(dir, 'rpc/pikku-rpc-wirings-meta.internal.gen.json'));
+    if (rpcMeta)
+        pikkuState(null, 'rpc', 'meta', rpcMeta);
+    const queueMeta = await readJson(logger, join(dir, 'queue/pikku-queue-workers-wirings-meta.gen.json'));
+    // Same reason as function meta: the workflow service adds its orchestrator /
+    // step queues (and wf-orchestrator-* / wf-step-* per-workflow queues) here at
+    // init, absent from the generated JSON — merge so they survive the reload.
+    if (queueMeta) {
+        const existing = pikkuState(null, 'queue', 'meta') ?? {};
+        pikkuState(null, 'queue', 'meta', { ...existing, ...queueMeta });
+    }
+    const agentMeta = await readJson(logger, join(dir, 'agent/pikku-agent-wirings-meta.gen.json'));
+    if (agentMeta?.agentsMeta) {
+        pikkuState(null, 'agent', 'agentsMeta', agentMeta.agentsMeta);
+    }
+    // Generated JSON schemas: <outDir>/schemas/schemas/<Name>.schema.json.
+    // Re-adding replaces the map entry; the schema service recompiles any
+    // validator whose stored schema value no longer matches.
+    const schemasDir = join(dir, 'schemas', 'schemas');
+    let schemaFiles = [];
+    try {
+        schemaFiles = await readdir(schemasDir);
+    }
+    catch {
+        // No generated schemas — a schema-less project.
+    }
+    for (const file of schemaFiles) {
+        if (!file.endsWith('.schema.json'))
+            continue;
+        const schema = await readJson(logger, join(schemasDir, file));
+        if (schema) {
+            addSchema(file.slice(0, -'.schema.json'.length), schema);
+        }
+    }
+    clearMiddlewareCache();
+    clearPermissionsCache();
+    clearChannelMiddlewareCache();
+    httpRouter.reset();
+    try {
+        compileAllSchemas(logger, schemaService);
+    }
+    catch (err) {
+        logger.error(`Hot-reload schema recompilation failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -509,13 +509,13 @@ export class PikkuWorkflowService {
     resolveStepJobOptions(stepOptions) {
         const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES;
         const retryDelay = stepOptions?.retryDelay;
-        const backoff = typeof retryDelay === 'number'
-            ? { type: 'fixed', delay: retryDelay }
-            : retryDelay === 'exponential'
+        // A concrete retryDelay (15000, '15s') is a fixed backoff; only the literal
+        // 'exponential' — or no delay at all — selects exponential.
+        const backoff = retryDelay !== undefined && retryDelay !== 'exponential'
+            ? { type: 'fixed', delay: getDurationInMilliseconds(retryDelay) }
+            : retries > 0 || retryDelay === 'exponential'
                 ? 'exponential'
-                : retries > 0
-                    ? 'exponential'
-                    : undefined;
+                : undefined;
         return { attempts: retries + 1, ...(backoff ? { backoff } : {}) };
     }
     async queueStepWorker(runId, stepName, rpcName, data, stepOptions, fromStepName) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.55",
+  "version": "0.12.57",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",

package/src/dev/hot-reload.test.ts

@@ -152,29 +152,36 @@ describe('pikkuDevReloader', { concurrency: false }, () => {
     assert.ok(reloadLog, 'Should log hot-reload message')
   })
 
-  test('should not replace a function that is not registered', async (t) => {
+  test('should register a brand-new function export', async (t) => {
     if (!(await ensureRecursiveWatchAvailable(t, tmpDir))) return
 
     addFunction('registeredFunc', {
       func: async () => ({ name: 'registered' }),
     })
 
-    await writeFunctionModule(tmpDir, 'unknownFunc.ts', '{ name: "unknown" }')
-
     reloader = await pikkuDevReloader({
       srcDirectories: [tmpDir],
       logger: mockLogger,
       pikkuDir: tmpDir,
     })
 
-    await writeFunctionModule(tmpDir, 'unknownFunc.ts', '{ name: "updated" }')
+    await writeFunctionModule(tmpDir, 'unknownFunc.ts', '{ name: "unknown" }')
 
     await wait(300)
 
-    assert.equal(
-      pikkuState(null, 'function', 'functions').has('unknownFunc'),
-      false
-    )
+    const func = pikkuState(null, 'function', 'functions').get('unknownFunc')!
+    assert.ok(func, 'New function export should be registered')
+    assert.deepEqual(await func.func({} as any, {}, {} as any), {
+      name: 'unknown',
+    })
+    const newLog = mockLogger
+      .getLogs()
+      .find(
+        (l) =>
+          l.message.includes('Hot-reloaded') &&
+          l.message.includes('new: unknownFunc')
+      )
+    assert.ok(newLog, 'Should log the newly registered function')
   })
 
   test('should keep old code when JS import fails', async (t) => {