@pikku/core 0.12.103 → 0.12.106

package/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.12.106
+
+### Patch Changes
+
+- bbb9e3b: Add a `scaffold.remoteJobs` flag that generates HTTP routes an external dispatcher posts to, so a runtime holding neither a queue consumer nor a clock still runs its queue workers and scheduled tasks. The runners and the secret guard live in `@pikku/core`; the scaffold only wires them.
+
+## 0.12.105
+
+### Patch Changes
+
+- 1ab831e: Fix hot reload silently keeping old code on Bun for any file importing a relative sibling.
+
+  Inside a `pikku dev` that has been running for a while, Bun stops honouring `createRequire`'s referrer: a file importing `./sibling.js` reloads as `Cannot find module './sibling.js'` even though it resolves fine in a fresh Bun script, so the reload aborts and the process goes on serving the previous version of a function whose source has visibly changed. Bun's own resolver still answers correctly when handed the importer's directory outright, so the module runner now resolves each specifier to an absolute path before requiring it, and the referrer never has to survive the trip. Specifiers Bun's resolver declines — builtins among them — fall through unchanged, and non-Bun runtimes keep the plain `createRequire`.
+
+## 0.12.104
+
+### Patch Changes
+
+- f10162a: `send()` now reports the delivery id it wrote, not only the broker's job id.
+
+  `KyselyWebhookService.send()` returned `{ jobId }` straight from `queueService.add()`, and callers handed that to `getDelivery()`. That only worked because every queue in the test suite echoed the requested `jobId` back. A broker is free to assign its own identity — the JetStream queue returns a stream sequence — so on those the read-back looked up an id that was never written and 404'd.
+
+  `SendWebhookResult` gains an optional `deliveryId`, which store-backed implementations always populate.
+
 ## 0.12.103
 
 ### Patch Changes

package/dist/dev/module-runner.js

@@ -18,6 +18,38 @@ const loadTransform = async () => {
  *  is correct, and no amount of re-reading it will reveal that the reloader —
  *  not the file — is what cannot cope. */
 export const isTopLevelAwaitLimitation = (error) => /top-level await/i.test(error.message);
+/** The `require` a reloaded module is run with.
+ *
+ *  On Bun, `createRequire`'s referrer stops being honoured somewhere in a
+ *  long-running process: the same relative specifier that resolves in a fresh
+ *  script fails inside a `pikku dev` that has been up for a while, so a file
+ *  importing `./sibling.js` reloads as `Cannot find module` and the developer
+ *  is left serving the previous code from an edit that looked applied. Bun's
+ *  own resolver still answers correctly when handed the importer's directory
+ *  outright, so the specifier is resolved to an absolute path first and the
+ *  referrer never has to survive the trip. Anything Bun's resolver declines —
+ *  builtins among them — falls through to `require` unchanged.
+ */
+const createRequireForFile = (absPath) => {
+    const base = createRequire(pathToFileURL(absPath));
+    const bun = globalThis.Bun;
+    if (!bun)
+        return base;
+    const importerDir = dirname(absPath);
+    const resolveThroughBun = (specifier) => {
+        try {
+            return bun.resolveSync(specifier, importerDir);
+        }
+        catch {
+            return specifier;
+        }
+    };
+    // Only resolution is delegated; the require itself stays outside the guard so
+    // a module that throws while evaluating reports its own error rather than
+    // being retried under a second specifier.
+    const bunRequire = ((specifier) => base(resolveThroughBun(specifier)));
+    return Object.assign(bunRequire, base);
+};
 export const createModuleRunner = () => {
     const registry = new Map();
     const run = async (filePath) => {
@@ -31,7 +63,7 @@ export const createModuleRunner = () => {
                 sourcefile: absPath,
             });
             const fn = compileFunction(code, ['require', 'exports', 'module', '__filename', '__dirname'], { filename: absPath });
-            const require = createRequire(pathToFileURL(absPath));
+            const require = createRequireForFile(absPath);
             const moduleObj = { exports: {} };
             fn(require, moduleObj.exports, moduleObj, absPath, dirname(absPath));
             registry.set(absPath, moduleObj.exports);

package/dist/middleware/index.d.ts

@@ -1,6 +1,7 @@
 export { authAPIKey } from './auth-apikey.js';
 export { authCookie } from './auth-cookie.js';
 export { authBearer } from './auth-bearer.js';
+export { remoteJobsSecret } from './remote-jobs-secret.js';
 export { pikkuRemoteAuthMiddleware } from './remote-auth.js';
 export { cors } from './cors.js';
 export { requireOrigin, isAllowedOrigin, toOrigin } from './require-origin.js';

package/dist/middleware/index.js

@@ -1,6 +1,7 @@
 export { authAPIKey } from './auth-apikey.js';
 export { authCookie } from './auth-cookie.js';
 export { authBearer } from './auth-bearer.js';
+export { remoteJobsSecret } from './remote-jobs-secret.js';
 export { pikkuRemoteAuthMiddleware } from './remote-auth.js';
 export { cors } from './cors.js';
 export { requireOrigin, isAllowedOrigin, toOrigin } from './require-origin.js';

package/dist/middleware/remote-jobs-secret.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Guards the remote job inbox. Fails closed: an unset secret rejects every
+ * caller, so the routes are never open just because the deployment forgot to
+ * configure one.
+ */
+export declare const remoteJobsSecret: import("./middleware.types.js").CorePikkuMiddleware<import("../types/core.types.js").CoreSingletonServices<{
+    logLevel?: import("../services/logger.js").LogLevel;
+    secrets?: {
+        requireAllowedHosts?: boolean;
+    };
+    workflow?: import("../wirings/workflow/workflow.types.js").WorkflowServiceConfig;
+    webhook?: import("../services/webhook-service.js").WebhookServiceConfig;
+    postgres?: import("../types/core.types.js").PostgresConfig;
+}>, import("../types/core.types.js").CoreUserSession>;

package/dist/middleware/remote-jobs-secret.js

@@ -0,0 +1,27 @@
+import { UnauthorizedError } from '../errors/errors.js';
+import { REMOTE_JOBS_SECRET_HEADER, REMOTE_JOBS_SECRET_VARIABLE, } from '../services/remote-jobs.js';
+import { pikkuMiddleware } from './middleware-factories.js';
+const constantTimeEqual = (a, b) => {
+    if (a.length !== b.length)
+        return false;
+    let result = 0;
+    for (let i = 0; i < a.length; i++) {
+        result |= a.charCodeAt(i) ^ b.charCodeAt(i);
+    }
+    return result === 0;
+};
+/**
+ * Guards the remote job inbox. Fails closed: an unset secret rejects every
+ * caller, so the routes are never open just because the deployment forgot to
+ * configure one.
+ */
+export const remoteJobsSecret = pikkuMiddleware(async ({ variables }, { http }, next) => {
+    const expected = await variables.get(REMOTE_JOBS_SECRET_VARIABLE);
+    const provided = http?.request?.header(REMOTE_JOBS_SECRET_HEADER);
+    if (!expected ||
+        typeof provided !== 'string' ||
+        !constantTimeEqual(provided, expected)) {
+        throw new UnauthorizedError('invalid dispatch secret');
+    }
+    return next();
+});

package/dist/services/credential-wire-service.d.ts

@@ -19,6 +19,21 @@ export declare class PikkuCredentialWireService {
     getAll(): Record<string, unknown> | Promise<Record<string, unknown>>;
     getScoped(allowedNames: string[]): Record<string, unknown> | Promise<Record<string, unknown>>;
     private lazyLoad;
+    /**
+     * `loaded` means "the credentials are in hand", never "a load has started".
+     *
+     * The distinction matters because `get`, `getAll` and `getScoped` all answer
+     * SYNCHRONOUSLY once it is set. Setting it before the await let every caller
+     * that arrived while the fetch was still in flight take that synchronous path
+     * over an empty map and receive `null` for a credential that exists — the
+     * first of N concurrent readers got the value and the rest silently did not.
+     * `loadPromise` is what guards against loading twice; this flag is only ever
+     * about whether the fast path is safe to take.
+     *
+     * `finally` rather than a trailing assignment: the two early returns (no
+     * service or wire, no resolvable user) are settled states as well — there is
+     * nothing to fetch — so they should reach the fast path too.
+     */
     private doLoad;
 }
 export declare function createMiddlewareCredentialWireProps(credentialWire: PikkuCredentialWireService): {

package/dist/services/credential-wire-service.js

@@ -56,20 +56,39 @@ export class PikkuCredentialWireService {
         this.loadPromise = this.doLoad();
         return this.loadPromise;
     }
+    /**
+     * `loaded` means "the credentials are in hand", never "a load has started".
+     *
+     * The distinction matters because `get`, `getAll` and `getScoped` all answer
+     * SYNCHRONOUSLY once it is set. Setting it before the await let every caller
+     * that arrived while the fetch was still in flight take that synchronous path
+     * over an empty map and receive `null` for a credential that exists — the
+     * first of N concurrent readers got the value and the rest silently did not.
+     * `loadPromise` is what guards against loading twice; this flag is only ever
+     * about whether the fast path is safe to take.
+     *
+     * `finally` rather than a trailing assignment: the two early returns (no
+     * service or wire, no resolvable user) are settled states as well — there is
+     * nothing to fetch — so they should reach the fast path too.
+     */
     async doLoad() {
-        this.loaded = true;
-        if (!this.credentialService || !this.wire)
-            return;
-        const userId = defaultPikkuUserIdResolver(this.wire);
-        if (!userId)
-            return;
-        this.wire.pikkuUserId = userId;
-        const allCreds = await this.credentialService.getAll(userId);
-        for (const [name, value] of Object.entries(allCreds)) {
-            if (!(name in this.credentials)) {
-                this.credentials[name] = value;
+        try {
+            if (!this.credentialService || !this.wire)
+                return;
+            const userId = defaultPikkuUserIdResolver(this.wire);
+            if (!userId)
+                return;
+            this.wire.pikkuUserId = userId;
+            const allCreds = await this.credentialService.getAll(userId);
+            for (const [name, value] of Object.entries(allCreds)) {
+                if (!(name in this.credentials)) {
+                    this.credentials[name] = value;
+                }
             }
         }
+        finally {
+            this.loaded = true;
+        }
     }
 }
 export function createMiddlewareCredentialWireProps(credentialWire) {

package/dist/services/index.d.ts

@@ -12,6 +12,7 @@ export { LocalVariablesService } from './local-variables.js';
 export { ConsoleLogger, JsonConsoleLogger } from './logger-console.js';
 export { InMemoryWorkflowService } from './in-memory-workflow-service.js';
 export { QueueWebhookService, pikkuWebhookWorkerFunc, } from './queue-webhook-service.js';
+export { REMOTE_JOBS_SECRET_HEADER, REMOTE_JOBS_SECRET_VARIABLE, REMOTE_QUEUE_JOB_PATH, REMOTE_SCHEDULER_JOB_PATH, pikkuRemoteQueueJobFunc, pikkuRemoteScheduledJobFunc, type RemoteQueueJobData, type RemoteScheduledJobData, } from './remote-jobs.js';
 export { InMemoryQueueService } from './in-memory-queue-service.js';
 export { InMemoryTriggerService } from './in-memory-trigger-service.js';
 export { InMemoryAgentRunStateService } from './in-memory-agent-run-state-service.js';

package/dist/services/index.js

@@ -12,6 +12,7 @@ export { LocalVariablesService } from './local-variables.js';
 export { ConsoleLogger, JsonConsoleLogger } from './logger-console.js';
 export { InMemoryWorkflowService } from './in-memory-workflow-service.js';
 export { QueueWebhookService, pikkuWebhookWorkerFunc, } from './queue-webhook-service.js';
+export { REMOTE_JOBS_SECRET_HEADER, REMOTE_JOBS_SECRET_VARIABLE, REMOTE_QUEUE_JOB_PATH, REMOTE_SCHEDULER_JOB_PATH, pikkuRemoteQueueJobFunc, pikkuRemoteScheduledJobFunc, } from './remote-jobs.js';
 export { InMemoryQueueService } from './in-memory-queue-service.js';
 export { InMemoryTriggerService } from './in-memory-trigger-service.js';
 export { InMemoryAgentRunStateService } from './in-memory-agent-run-state-service.js';

package/dist/services/remote-jobs.d.ts

@@ -0,0 +1,27 @@
+import type { Logger } from './logger.js';
+/** The variable holding the shared secret every remote job must present. */
+export declare const REMOTE_JOBS_SECRET_VARIABLE = "PIKKU_DISPATCH_SECRET";
+export declare const REMOTE_JOBS_SECRET_HEADER = "x-pikku-dispatch";
+export declare const REMOTE_QUEUE_JOB_PATH = "/__pikku/queue-job";
+export declare const REMOTE_SCHEDULER_JOB_PATH = "/__pikku/scheduler-job";
+export interface RemoteQueueJobData {
+    queueName: string;
+    data?: unknown;
+    jobId?: string;
+    traceId?: string;
+}
+export interface RemoteScheduledJobData {
+    taskName: string;
+}
+/**
+ * A dispatched job is delivered, not owned: the sender retries, so a permanent
+ * failure has to read differently from a transient one. An unknown worker or a
+ * discarded job answers 4xx so the sender stops; anything else propagates and
+ * is delivered again.
+ */
+export declare function pikkuRemoteQueueJobFunc({ logger }: {
+    logger: Logger;
+}, { queueName, data, jobId, traceId }: RemoteQueueJobData): Promise<void>;
+export declare function pikkuRemoteScheduledJobFunc({ logger }: {
+    logger: Logger;
+}, { taskName }: RemoteScheduledJobData): Promise<void>;

package/dist/services/remote-jobs.js

@@ -0,0 +1,59 @@
+import { BadRequestError, PikkuMissingMetaError } from '../errors/errors.js';
+import { QueueJobDiscardedError, runQueueJob } from '../wirings/queue/queue-runner.js';
+import { ScheduledTaskNotFoundError, runScheduledTask, } from '../wirings/scheduler/scheduler-runner.js';
+/** The variable holding the shared secret every remote job must present. */
+export const REMOTE_JOBS_SECRET_VARIABLE = 'PIKKU_DISPATCH_SECRET';
+export const REMOTE_JOBS_SECRET_HEADER = 'x-pikku-dispatch';
+export const REMOTE_QUEUE_JOB_PATH = '/__pikku/queue-job';
+export const REMOTE_SCHEDULER_JOB_PATH = '/__pikku/scheduler-job';
+/**
+ * A dispatched job is delivered, not owned: the sender retries, so a permanent
+ * failure has to read differently from a transient one. An unknown worker or a
+ * discarded job answers 4xx so the sender stops; anything else propagates and
+ * is delivered again.
+ */
+export async function pikkuRemoteQueueJobFunc({ logger }, { queueName, data, jobId, traceId }) {
+    const job = {
+        queueName,
+        data,
+        id: jobId ?? traceId ?? `remote-${Date.now()}`,
+        status: async () => 'active',
+        metadata: () => ({
+            processedAt: new Date(),
+            attemptsMade: 0,
+            maxAttempts: undefined,
+            result: undefined,
+            progress: 0,
+            createdAt: new Date(),
+            completedAt: undefined,
+            failedAt: undefined,
+            error: undefined,
+        }),
+        waitForCompletion: async () => {
+            throw new Error('remote jobs do not support waitForCompletion');
+        },
+    };
+    try {
+        await runQueueJob({ job, traceId });
+    }
+    catch (error) {
+        if (error instanceof PikkuMissingMetaError ||
+            error instanceof QueueJobDiscardedError) {
+            throw new BadRequestError(error.message);
+        }
+        logger.error(`remote-jobs: queue '${queueName}' failed: ${error instanceof Error ? error.message : String(error)}`);
+        throw error;
+    }
+}
+export async function pikkuRemoteScheduledJobFunc({ logger }, { taskName }) {
+    try {
+        await runScheduledTask({ name: taskName, traceId: `cron-${taskName}` });
+    }
+    catch (error) {
+        if (error instanceof ScheduledTaskNotFoundError) {
+            throw new BadRequestError(error.message);
+        }
+        logger.error(`remote-jobs: task '${taskName}' failed: ${error instanceof Error ? error.message : String(error)}`);
+        throw error;
+    }
+}

package/dist/services/webhook-service.d.ts

@@ -16,6 +16,14 @@ export interface SendWebhookInput {
 }
 export interface SendWebhookResult {
     jobId: string;
+    /**
+     * The delivery row's id, which is what `getDelivery` reads back. Present only
+     * on store-backed implementations. It is not interchangeable with `jobId`: a
+     * broker is free to assign its own identity — the JetStream queue returns a
+     * stream sequence — so a caller that hands `jobId` to `getDelivery` looks up
+     * an id that was never written.
+     */
+    deliveryId?: string;
 }
 export interface WebhookServiceConfig {
     retries?: number;

package/dist/wirings/scheduler/index.d.ts

@@ -1,3 +1,3 @@
-export { runScheduledTask, wireScheduler, getScheduledTasks, } from './scheduler-runner.js';
+export { ScheduledTaskNotFoundError, runScheduledTask, wireScheduler, getScheduledTasks, } from './scheduler-runner.js';
 export { logSchedulers } from './log-schedulers.js';
 export type { CoreScheduledTask, ScheduledTasksMeta, } from './scheduler.types.js';

package/dist/wirings/scheduler/index.js

@@ -1,2 +1,2 @@
-export { runScheduledTask, wireScheduler, getScheduledTasks, } from './scheduler-runner.js';
+export { ScheduledTaskNotFoundError, runScheduledTask, wireScheduler, getScheduledTasks, } from './scheduler-runner.js';
 export { logSchedulers } from './log-schedulers.js';

package/dist/wirings/scheduler/scheduler-runner.d.ts

@@ -1,5 +1,6 @@
 import type { CoreUserSession } from '../../types/core.types.js';
 import type { CoreScheduledTask } from './scheduler.types.js';
+import { PikkuError } from '../../errors/error-handler.js';
 import type { CorePikkuFunctionConfig, CorePikkuFunctionSessionless } from '../../function/functions.types.js';
 export type RunScheduledTasksParams = {
     name: string;
@@ -7,5 +8,8 @@ export type RunScheduledTasksParams = {
     traceId?: string;
 };
 export declare const wireScheduler: <PikkuFunctionConfig extends CorePikkuFunctionConfig<CorePikkuFunctionSessionless<void, void>>>(scheduledTask: CoreScheduledTask<PikkuFunctionConfig>) => void;
+export declare class ScheduledTaskNotFoundError extends PikkuError {
+    constructor(title: string);
+}
 export declare function runScheduledTask({ name, session, traceId, }: RunScheduledTasksParams): Promise<void>;
 export declare const getScheduledTasks: () => Map<string, CoreScheduledTask>;

package/dist/wirings/scheduler/scheduler-runner.js

@@ -22,7 +22,7 @@ export const wireScheduler = (scheduledTask) => {
     }
     tasks.set(scheduledTask.name, scheduledTask);
 };
-class ScheduledTaskNotFoundError extends PikkuError {
+export class ScheduledTaskNotFoundError extends PikkuError {
     constructor(title) {
         super(`Scheduled task not found: ${title}`);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.103",
+  "version": "0.12.106",
   "description": "The Pikku runtime — functions, wirings, services, middleware and types",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",

package/src/dev/module-runner.test.ts

@@ -209,4 +209,27 @@ describe('createModuleRunner', { concurrency: false }, () => {
       `heap grew ${growth.toFixed(1)} MB over 200 edits (expected < 15 MB)`
     )
   })
+
+  test('a .js specifier resolves to the .ts file that actually exists', async () => {
+    // How every TypeScript source in a pikku app refers to a sibling: the
+    // import is written `.js` and only the `.ts` is on disk. Nothing else here
+    // covers it — the delegated-import test writes a real `.js` — and it is the
+    // shape that breaks under Bun once `pikku dev` has been running, where the
+    // reload fails and the process quietly keeps serving the previous code.
+    await writeFile(join(tmpDir, 'dep.ts'), `export const value = 'from-dep'`)
+
+    const runner = createModuleRunner()
+    const userFile = join(tmpDir, 'importer.ts')
+    await writeFile(
+      userFile,
+      `import { value } from './dep.js'
+       export const fn = { func: async () => value }`
+    )
+
+    const result = await runner.run(userFile)
+    assert.equal(result.ok, true)
+    const mod = (result as { exports: Record<string, unknown> }).exports
+    assert.equal(await (mod.fn as { func: () => Promise<string> }).func(), 'from-dep')
+  })
+
 })

package/src/dev/module-runner.ts

@@ -43,6 +43,44 @@ export interface PikkuModuleRunner {
 export const isTopLevelAwaitLimitation = (error: Error): boolean =>
   /top-level await/i.test(error.message)
 
+interface BunResolver {
+  resolveSync: (specifier: string, parent: string) => string
+}
+
+/** The `require` a reloaded module is run with.
+ *
+ *  On Bun, `createRequire`'s referrer stops being honoured somewhere in a
+ *  long-running process: the same relative specifier that resolves in a fresh
+ *  script fails inside a `pikku dev` that has been up for a while, so a file
+ *  importing `./sibling.js` reloads as `Cannot find module` and the developer
+ *  is left serving the previous code from an edit that looked applied. Bun's
+ *  own resolver still answers correctly when handed the importer's directory
+ *  outright, so the specifier is resolved to an absolute path first and the
+ *  referrer never has to survive the trip. Anything Bun's resolver declines —
+ *  builtins among them — falls through to `require` unchanged.
+ */
+const createRequireForFile = (absPath: string): NodeRequire => {
+  const base = createRequire(pathToFileURL(absPath))
+  const bun = (globalThis as { Bun?: BunResolver }).Bun
+  if (!bun) return base
+
+  const importerDir = dirname(absPath)
+  const resolveThroughBun = (specifier: string): string => {
+    try {
+      return bun.resolveSync(specifier, importerDir)
+    } catch {
+      return specifier
+    }
+  }
+
+  // Only resolution is delegated; the require itself stays outside the guard so
+  // a module that throws while evaluating reports its own error rather than
+  // being retried under a second specifier.
+  const bunRequire = ((specifier: string) =>
+    base(resolveThroughBun(specifier))) as NodeRequire
+  return Object.assign(bunRequire, base)
+}
+
 export const createModuleRunner = (): PikkuModuleRunner => {
   const registry = new Map<string, Record<string, unknown>>()
 
@@ -63,7 +101,7 @@ export const createModuleRunner = (): PikkuModuleRunner => {
         { filename: absPath }
       )
 
-      const require = createRequire(pathToFileURL(absPath))
+      const require = createRequireForFile(absPath)
       const moduleObj: { exports: Record<string, unknown> } = { exports: {} }
       fn(require, moduleObj.exports, moduleObj, absPath, dirname(absPath))
 

package/src/middleware/index.ts

@@ -1,6 +1,7 @@
 export { authAPIKey } from './auth-apikey.js'
 export { authCookie } from './auth-cookie.js'
 export { authBearer } from './auth-bearer.js'
+export { remoteJobsSecret } from './remote-jobs-secret.js'
 export { pikkuRemoteAuthMiddleware } from './remote-auth.js'
 export { cors } from './cors.js'
 export { requireOrigin, isAllowedOrigin, toOrigin } from './require-origin.js'

package/src/middleware/remote-jobs-secret.ts

@@ -0,0 +1,35 @@
+import { UnauthorizedError } from '../errors/errors.js'
+import {
+  REMOTE_JOBS_SECRET_HEADER,
+  REMOTE_JOBS_SECRET_VARIABLE,
+} from '../services/remote-jobs.js'
+import { pikkuMiddleware } from './middleware-factories.js'
+
+const constantTimeEqual = (a: string, b: string): boolean => {
+  if (a.length !== b.length) return false
+  let result = 0
+  for (let i = 0; i < a.length; i++) {
+    result |= a.charCodeAt(i) ^ b.charCodeAt(i)
+  }
+  return result === 0
+}
+
+/**
+ * Guards the remote job inbox. Fails closed: an unset secret rejects every
+ * caller, so the routes are never open just because the deployment forgot to
+ * configure one.
+ */
+export const remoteJobsSecret = pikkuMiddleware(
+  async ({ variables }, { http }, next) => {
+    const expected = await variables.get(REMOTE_JOBS_SECRET_VARIABLE)
+    const provided = http?.request?.header(REMOTE_JOBS_SECRET_HEADER)
+    if (
+      !expected ||
+      typeof provided !== 'string' ||
+      !constantTimeEqual(provided, expected)
+    ) {
+      throw new UnauthorizedError('invalid dispatch secret')
+    }
+    return next()
+  }
+)