@pikku/core 0.12.77 → 0.12.79

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -142,6 +142,24 @@ const WORKFLOW_TERMINAL_STATES = new Set([
     'failed',
     'cancelled',
 ]);
+/** Idle window before a `running` run with nothing in flight is treated as stalled. */
+const DEFAULT_STALLED_RUN_MS = 5 * 60_000;
+/** Runs re-driven per `recoverStalledRuns` call, so one sweep is bounded. */
+const DEFAULT_STALLED_RUN_LIMIT = 100;
+/**
+ * How long a step may sit `pending` before the relay assumes its dispatch was
+ * lost. This is a bet that no healthy queue takes this long to move a job from
+ * `pending` to `running`; set it above the observed p99 of that latency.
+ */
+const DEFAULT_UNDISPATCHED_STEP_MS = 30_000;
+/** Steps re-driven per `relayUndispatchedSteps` call, so one tick is bounded. */
+const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
+/** First wait before a step already re-dispatched once is re-dispatched again. */
+const REDISPATCH_BACKOFF_MS = 30_000;
+/** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */
+const REDISPATCH_BACKOFF_MAX_MS = 10 * 60_000;
+/** Bound on the in-process backoff map, so a long-lived process cannot grow it without bound. */
+const REDISPATCH_BACKOFF_MAX_ENTRIES = 10_000;
 const WORKFLOW_POLL_MIN_MS = 10;
 const WORKFLOW_POLL_FACTOR = 1.6;
 const WORKFLOW_CHILD_POLL_MAX_MS = 500;
@@ -435,6 +453,143 @@ export class PikkuWorkflowService {
             group: this.getJobGroup(workflowName),
         });
     }
+    /**
+     * Ids of runs that are stalled: still `running`, with no step in a state that
+     * something is expected to complete (`running`, `scheduled`, `suspended`),
+     * and no step activity since `before`.
+     *
+     * Returns nothing by default so a store that cannot express the query keeps
+     * working unchanged; a store that overrides it gains crash recovery through
+     * `recoverStalledRuns`.
+     */
+    async findStalledRunIds(_before, _limit) {
+        return [];
+    }
+    /**
+     * Re-drive runs whose next move was lost, and report which were resumed.
+     *
+     * Arming a step is two writes to two systems — the step row, then the queue
+     * or scheduler job — so a process that dies between them leaves a run that is
+     * `running` with nothing in flight. Nothing notices: the run parks on a step
+     * that will never complete and never error, so it neither finishes nor fails.
+     * (Seen on a `workflow.sleep()`: a deploy restart landed between the sleep
+     * step's insert and its timer, parking the run permanently.)
+     *
+     * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted
+     * step state, and every settled step is memoized, so resuming a run that was
+     * not actually stuck costs an orchestration pass and changes nothing. That
+     * idempotence is what makes an idle-time heuristic safe here; a run that is
+     * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+     *
+     * This is not self-starting. Call it from a scheduled task at whatever
+     * interval suits the workload.
+     */
+    async recoverStalledRuns(options) {
+        const before = new Date(Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS));
+        const runIds = await this.findStalledRunIds(before, options?.limit ?? DEFAULT_STALLED_RUN_LIMIT);
+        const resumed = [];
+        for (const runId of runIds) {
+            try {
+                await this.resumeWorkflow(runId);
+                resumed.push(runId);
+            }
+            catch (err) {
+                // One unresumable run must not stop the sweep from recovering the rest.
+                getSingletonServices()?.logger?.error(`Failed to resume stalled workflow run ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return { resumed };
+    }
+    /**
+     * Runs holding a step that has sat `pending` since before `before`, paired
+     * with the step that flagged them.
+     *
+     * Returns nothing by default so a store that cannot express the query keeps
+     * working unchanged — and, because it does not opt in, gains no re-dispatches
+     * either. A store must have an atomic `withStepLock` before overriding this,
+     * or no concurrency for one to exclude: the relay makes duplicate dispatch
+     * routine, and the claim in `executeWorkflowStepInner` is what keeps a
+     * duplicate from becoming a second execution. `kysely-postgres` and
+     * `kysely-mysql` qualify on the lock, `in-memory` on being inline and
+     * single-process; `mongodb` and `kysely-sqlite` qualify on neither.
+     */
+    async findUndispatchedSteps(_before, _limit) {
+        return [];
+    }
+    /**
+     * Re-drive steps whose dispatch was lost, and report which runs were nudged.
+     *
+     * Arming a step is two writes to two systems: the step row lands `pending`,
+     * then a queue or scheduler job is published. Nothing spans both, so a crash
+     * in between leaves a durable row that nothing will ever pick up — the run
+     * neither finishes nor fails. (Seen on a `workflow.sleep()`: a deploy restart
+     * landed between the sleep step's insert and its timer.)
+     *
+     * The row is the outbox record and this is the relay. Age is the only signal
+     * available — a step `pending` because its dispatch was lost is
+     * indistinguishable from one whose job is merely still queued — so a step
+     * past `undispatchedAfterMs` is re-dispatched regardless, and correctness
+     * rests on the claim rather than on the guess being right. A redundant
+     * dispatch costs one queue message: the loser reads `running` and returns
+     * without invoking anything.
+     *
+     * Re-dispatches back off per step (doubling from 30s, capped at 10m) so a
+     * genuine queue backlog is not amplified by a tick that keeps firing at the
+     * steps the backlog is already delaying. The backoff is per process and
+     * advisory — losing it on restart costs extra dispatches, never correctness.
+     *
+     * This is not self-starting. Call it from a scheduled task; ~30s suits a
+     * queue whose `pending`→`running` latency is well under that.
+     */
+    async relayUndispatchedSteps(options) {
+        const before = new Date(Date.now() -
+            (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS));
+        const steps = await this.findUndispatchedSteps(before, options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT);
+        // Backoff is keyed by run, not step, because the run is the unit of
+        // re-drive: `resumeWorkflow` replays the whole run and re-dispatches every
+        // step still owed a job. Holding off a single step while resuming its run
+        // would not suppress anything.
+        const now = Date.now();
+        const runIds = new Set();
+        for (const { runId } of steps) {
+            const eligibleAt = this.redispatchBackoff.get(runId);
+            if (eligibleAt !== undefined && eligibleAt > now) {
+                continue;
+            }
+            this.noteRedispatch(runId, now);
+            runIds.add(runId);
+        }
+        const redispatched = [];
+        for (const runId of runIds) {
+            try {
+                await this.resumeWorkflow(runId);
+                redispatched.push(runId);
+            }
+            catch (err) {
+                // One unresumable run must not stop the tick from relaying the rest.
+                getSingletonServices()?.logger?.error(`Failed to re-dispatch workflow run ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return { redispatched };
+    }
+    /** Advisory, per-process record of when a run may next be re-dispatched. */
+    redispatchBackoff = new Map();
+    redispatchDelays = new Map();
+    noteRedispatch(runId, now) {
+        const previous = this.redispatchDelays.get(runId);
+        const delay = Math.min(previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2, REDISPATCH_BACKOFF_MAX_MS);
+        // A run that settles is never returned again, so entries are only evicted by
+        // this bound — oldest first, which is also least recently re-dispatched.
+        if (this.redispatchBackoff.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
+            const oldest = this.redispatchBackoff.keys().next();
+            if (!oldest.done) {
+                this.redispatchBackoff.delete(oldest.value);
+                this.redispatchDelays.delete(oldest.value);
+            }
+        }
+        this.redispatchDelays.set(runId, delay);
+        this.redispatchBackoff.set(runId, now + delay);
+    }
     resolveStepJobOptions(stepOptions) {
         const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES;
         const retryDelay = stepOptions?.retryDelay;

package/knowledge/decisions/security/gateway-handlers-run-through-the-function-runner-gate.md

@@ -17,7 +17,16 @@ runner's gate is the only thing that evaluates a function's declared `auth`,
 `scopes` and `permissions`; a direct call runs the handler with none of them
 checked.
 
-The handler is registered with `sessionless: true`. Gateway inbound traffic is
+The synthetic registration **inherits the wired function's own metadata** —
+`sessionless`, input and output schema names, `scopes`, tag middleware — read
+back from `pikkuState(null, 'gateway', 'meta')[name].pikkuFuncId`, which the
+inspector already records. The synthetic id exists so the gate runs; it is not a
+licence to run the handler under metadata its author never wrote. A handler
+declared with `pikkuFunc` says session-required in meta rather than through an
+`auth` property, and that declaration is honoured.
+
+When nothing was declared — no inspector entry, as when a gateway is wired by
+hand — the handler falls back to `sessionless: true`. Gateway inbound traffic is
 authenticated by the platform adapter (webhook signature verification, platform
 tokens), not by a user session, so defaulting to session-required would reject
 every legitimate webhook. `CoreGateway.auth` and the handler's own `auth: true`
@@ -26,6 +35,7 @@ whenever declared, session or not.
 
 **What this rules out:** invoking `config.func` directly from any gateway
 transport as an optimisation, and "simplifying" the synthetic function
-registration away. It also rules out flipping the sessionless default to
-session-required as a hardening measure — that breaks every webhook rather than
+registration away. It also rules out fabricating the handler's metadata rather
+than inheriting it, and flipping the _fallback_ to session-required as a
+hardening measure — that breaks every webhook that declared nothing, rather than
 securing it; require auth per gateway via `CoreGateway.auth` instead.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.77",
+  "version": "0.12.79",
   "description": "The Pikku runtime — functions, wirings, services, middleware and types",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
@@ -37,6 +37,7 @@
     "./trigger": "./dist/wirings/trigger/index.js",
     "./rpc": "./dist/wirings/rpc/index.js",
     "./safe-fetch": "./dist/utils/safe-fetch.js",
+    "./node-host-resolver": "./dist/utils/node-host-resolver.js",
     "./mcp": "./dist/wirings/mcp/index.js",
     "./ai-agent": "./dist/wirings/ai-agent/index.js",
     "./gateway": "./dist/wirings/gateway/index.js",

package/src/services/in-memory-workflow-service.ts

@@ -301,6 +301,58 @@ export class InMemoryWorkflowService
     return newStep
   }
 
+  /**
+   * Safe to opt in despite the pass-through `withStepLock`: this service wires
+   * no queues and runs every step inline in one process, so the relay's
+   * redundant dispatch has no second holder to race with. See
+   * knowledge: decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md
+   */
+  protected async findUndispatchedSteps(
+    before: Date,
+    limit: number
+  ): Promise<Array<{ runId: string; stepId: string }>> {
+    const undispatched: Array<{ runId: string; stepId: string }> = []
+    for (const [runId, run] of this.runs) {
+      if (run.status !== 'running') continue
+      for (const step of this.stepHistory.get(runId) ?? []) {
+        if (step.status !== 'pending') continue
+        if (step.updatedAt >= before) continue
+        undispatched.push({ runId, stepId: step.stepId })
+        if (undispatched.length >= limit) return undispatched
+      }
+    }
+    return undispatched
+  }
+
+  protected async findStalledRunIds(
+    before: Date,
+    limit: number
+  ): Promise<string[]> {
+    const stalled: string[] = []
+    for (const [runId, run] of this.runs) {
+      if (run.status !== 'running') continue
+      const steps = this.stepHistory.get(runId) ?? []
+      if (
+        steps.some(
+          (step) =>
+            step.status === 'running' ||
+            step.status === 'scheduled' ||
+            step.status === 'suspended'
+        )
+      ) {
+        continue
+      }
+      const lastActivity = steps.reduce(
+        (latest, step) => (step.updatedAt > latest ? step.updatedAt : latest),
+        run.updatedAt
+      )
+      if (lastActivity >= before) continue
+      stalled.push(runId)
+      if (stalled.length >= limit) break
+    }
+    return stalled
+  }
+
   async listRuns(options?: {
     workflowName?: string
     status?: string

package/src/utils/node-host-resolver.ts

@@ -0,0 +1,20 @@
+import { lookup } from 'node:dns/promises'
+
+import { setDefaultHostResolver, type HostResolver } from './safe-fetch.js'
+
+/**
+ * Resolves a hostname through the platform resolver, returning every address it
+ * points at so `safeFetch` can reject a public name aimed at an internal one.
+ *
+ * This module is Node-only and is never imported by core itself — Workers has no
+ * DNS API, and a static `node:dns` import in the shared path would break that
+ * build.
+ */
+export const nodeHostResolver: HostResolver = async (hostname) => {
+  const results = await lookup(hostname, { all: true, verbatim: true })
+  return results.map(({ address }) => address)
+}
+
+/** Installs {@link nodeHostResolver} as the default for every `safeFetch`. */
+export const installNodeHostResolver = () =>
+  setDefaultHostResolver(nodeHostResolver)

package/src/utils/safe-fetch.test.ts

@@ -41,15 +41,54 @@ describe('isPrivateHost', () => {
     }
   })
 
+  test('flags reserved IPv4 blocks that are not routable public space', () => {
+    for (const host of [
+      '100.64.0.1', // carrier-grade NAT
+      '100.100.100.200', // Alibaba Cloud metadata endpoint
+      '100.127.255.255', // top of 100.64.0.0/10
+      '192.0.0.192', // IETF protocol assignments
+      '198.18.0.1', // benchmarking
+      '198.19.255.255', // top of 198.18.0.0/15
+      '192.88.99.1', // 6to4 anycast
+      '192.0.2.1', // TEST-NET-1
+      '198.51.100.1', // TEST-NET-2
+      '203.0.113.1', // TEST-NET-3
+      '224.0.0.1', // multicast
+      '240.0.0.1', // reserved
+      '255.255.255.255', // broadcast
+    ]) {
+      assert.equal(isPrivateHost(host), true, `${host} should be private`)
+    }
+  })
+
+  test('flags IPv6 forms that tunnel to an internal IPv4 address', () => {
+    for (const host of [
+      '64:ff9b::a9fe:a9fe', // NAT64 wrapping 169.254.169.254
+      '[64:ff9b::169.254.169.254]', // same, dotted-quad tail
+      '2002:a9fe:a9fe::', // 6to4 wrapping 169.254.169.254
+      '2002:6464:64c8::', // 6to4 wrapping 100.100.100.200
+      'fec0::1', // deprecated site-local
+      'ff02::1', // multicast
+      '100::1', // discard-only
+    ]) {
+      assert.equal(isPrivateHost(host), true, `${host} should be private`)
+    }
+  })
+
   test('allows public hosts', () => {
     for (const host of [
       'example.com',
       '8.8.8.8',
       '172.32.0.1',
       '11.0.0.1',
+      '100.63.255.255', // just below the CGNAT block
+      '100.128.0.0', // just above the CGNAT block
+      '198.17.255.255', // just below the benchmarking block
+      '198.20.0.0', // just above the benchmarking block
       '134744072', // decimal-encoded 8.8.8.8 — public
       '2001:db8::1', // documentation range — public
-      'fec0::1', // deprecated site-local, outside fe80::/10 — treated public
+      '2002:808:808::', // 6to4 wrapping public 8.8.8.8
+      '64:ff9b::808:808', // NAT64 wrapping public 8.8.8.8
     ]) {
       assert.equal(isPrivateHost(host), false, `${host} should be public`)
     }
@@ -138,6 +177,109 @@ describe('safeFetch', () => {
     )
   })
 
+  test('refuses a public hostname that resolves to a private address', async () => {
+    await withStubbedFetch(
+      () => new Response('ok', { status: 200 }),
+      async (calls) => {
+        // The shape the literal-only check misses: a name that parses as public
+        // but whose A record points at the cloud metadata endpoint.
+        await assert.rejects(
+          safeFetch(
+            'https://169-254-169-254.nip.io/latest/meta-data/',
+            {},
+            {
+              resolveHost: async () => ['169.254.169.254'],
+            }
+          ),
+          /resolves to 169\.254\.169\.254/
+        )
+        assert.equal(calls.length, 0)
+      }
+    )
+  })
+
+  test('refuses when any one resolved address is private', async () => {
+    await withStubbedFetch(
+      () => new Response('ok', { status: 200 }),
+      async (calls) => {
+        await assert.rejects(
+          safeFetch(
+            'https://split-horizon.example.com/',
+            {},
+            {
+              resolveHost: async () => ['93.184.216.34', '10.0.0.5'],
+            }
+          ),
+          /resolves to 10\.0\.0\.5/
+        )
+        assert.equal(calls.length, 0)
+      }
+    )
+  })
+
+  test('re-resolves the host of a redirect hop', async () => {
+    await withStubbedFetch(
+      (url) =>
+        url.includes('start.com')
+          ? new Response(null, {
+              status: 302,
+              headers: { location: 'https://rebound.example.com/next' },
+            })
+          : new Response('ok', { status: 200 }),
+      async (calls) => {
+        await assert.rejects(
+          safeFetch(
+            'https://start.com',
+            {},
+            {
+              resolveHost: async (hostname) =>
+                hostname === 'start.com' ? ['93.184.216.34'] : ['127.0.0.1'],
+            }
+          ),
+          /resolves to 127\.0\.0\.1/
+        )
+        assert.equal(calls.length, 1)
+      }
+    )
+  })
+
+  test('allows a public hostname that resolves to public addresses', async () => {
+    await withStubbedFetch(
+      () => new Response('ok', { status: 200 }),
+      async (calls) => {
+        const res = await safeFetch(
+          'https://example.com/',
+          {},
+          {
+            resolveHost: async () => ['93.184.216.34'],
+          }
+        )
+        assert.equal(res.status, 200)
+        assert.equal(calls.length, 1)
+      }
+    )
+  })
+
+  test('does not resolve an IP literal or an allowlisted host', async () => {
+    await withStubbedFetch(
+      () => new Response('ok', { status: 200 }),
+      async () => {
+        let resolverCalls = 0
+        const resolveHost = async () => {
+          resolverCalls++
+          return ['10.0.0.1']
+        }
+        await safeFetch('https://93.184.216.34/', {}, { resolveHost })
+        await safeFetch(
+          'https://internal.example.com/',
+          {},
+          { resolveHost, allowedHosts: ['internal.example.com'] }
+        )
+        assert.equal(resolverCalls, 0)
+      }
+    )
+  })
+
   test('follows a redirect to another public host', async () => {
     await withStubbedFetch(
       (url) =>