@pikku/core 0.12.86 → 0.12.88

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -9,13 +9,14 @@ import { deriveInvocationId } from './workflow-invocation-id.js';
 import { approvalDeciderFrom } from './workflow-approval-policy.js';
 import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
 import { DEFAULT_STEP_RETRIES, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, } from './workflow-constants.js';
-import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepFunctionMismatchError, WorkflowStepNameNotString, WorkflowSuspendedException, } from './workflow-errors.js';
+import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowSuspendedException, } from './workflow-errors.js';
 import { resolveWorkflowMeta } from './workflow-meta-resolver.js';
 import { jobGroupFor, orchestratorQueueName, resolveWorkflowConfig, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
 import { wireWorkflowQueueWorkers } from './workflow-queue-wiring.js';
 import { approvalStepNameFor, evaluateApprovalStep, recordApprovalDecision, } from './workflow-approval.js';
 import { auditApprovalDecision } from './workflow-approval-audit.js';
 import { recordSuspension, suspendStepNameFor } from './workflow-suspend.js';
+import { claimStepByReadThenWrite } from './workflow-step-claim.js';
 import { RedispatchBackoff, sweepStalledRuns, sweepUndispatchedSteps, } from './workflow-recovery.js';
 export class PikkuWorkflowService {
     runExtension;
@@ -261,12 +262,12 @@ export class PikkuWorkflowService {
      *
      * Returns nothing by default so a store that cannot express the query keeps
      * working unchanged — and, because it does not opt in, gains no re-dispatches
-     * either. A store must have an atomic `withStepLock` before overriding this,
-     * or no concurrency for one to exclude: the relay makes duplicate dispatch
-     * routine, and the claim in `executeWorkflowStepInner` is what keeps a
-     * duplicate from becoming a second execution. `kysely-postgres` and
-     * `kysely-mysql` qualify on the lock, `in-memory` on being inline and
-     * single-process; `mongodb` and `kysely-sqlite` qualify on neither.
+     * either. A store must have an atomic `claimStepForExecution` before
+     * overriding this, or no concurrency for one to exclude: the relay makes
+     * duplicate dispatch routine, and the claim is what keeps a duplicate from
+     * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
+     * status-guarded claim, `in-memory` on being inline and single-process;
+     * `mongodb` still qualifies on neither.
      */
     async findUndispatchedSteps(_before, _limit) {
         return [];
@@ -697,25 +698,24 @@ export class PikkuWorkflowService {
             this.exitExecution(runId);
         }
     }
+    /**
+     * Take sole ownership of a step before it runs, returning the state to run
+     * under — or `null` when another dispatch already owns it.
+     *
+     * Dispatch is at-least-once by design: the relay re-dispatches steps it
+     * believes were dropped, and a queue can redeliver a job it already handed
+     * out. This is the one place that keeps a duplicate dispatch from becoming a
+     * second execution of a side-effecting step, so it is only as strong as the
+     * exclusion it is built on — and `withStepLock` excludes nothing unless the
+     * store backs it with a real primitive. A store able to express the decision
+     * as one conditional write should override this rather than reach for a lock,
+     * which is what `@pikku/kysely` does with a status-guarded `UPDATE`.
+     */
+    async claimStepForExecution(runId, stepName, rpcName) {
+        return this.withStepLock(runId, stepName, () => claimStepByReadThenWrite(this, runId, stepName, rpcName));
+    }
     async executeWorkflowStepInner(runId, stepName, rpcName, data, rpcService) {
-        const claimed = await this.withStepLock(runId, stepName, async () => {
-            const stepState = await this.getStepState(runId, stepName);
-            // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
-            if (stepState.rpcName !== undefined &&
-                stepState.rpcName !== (rpcName ?? null)) {
-                throw new WorkflowStepFunctionMismatchError(runId, stepName);
-            }
-            if (stepState.status === 'succeeded' || stepState.status === 'running') {
-                return null;
-            }
-            if (stepState.status === 'failed') {
-                return this.createRetryAttempt(stepState.stepId, 'running');
-            }
-            if (stepState.status === 'pending' || stepState.status === 'scheduled') {
-                await this.setStepRunning(stepState.stepId);
-            }
-            return stepState;
-        });
+        const claimed = await this.claimStepForExecution(runId, stepName, rpcName);
         if (!claimed) {
             return;
         }

package/dist/wirings/workflow/workflow-step-claim.d.ts

@@ -0,0 +1,16 @@
+import type { StepState } from './workflow.types.js';
+/** What claiming a step needs from the workflow service. */
+export type StepClaimStore = {
+    getStepState(runId: string, stepName: string): Promise<StepState>;
+    setStepRunning(stepId: string): Promise<void>;
+    createRetryAttempt(failedStepId: string, status: 'pending' | 'running'): Promise<StepState>;
+};
+/**
+ * Decide whether this dispatch owns the step, by reading its state and then
+ * writing it — which only excludes a concurrent dispatch when the caller holds
+ * a lock that genuinely excludes one.
+ *
+ * A store that can express the whole decision as a single conditional write
+ * should do that instead of calling this.
+ */
+export declare const claimStepByReadThenWrite: (store: StepClaimStore, runId: string, stepName: string, rpcName: string) => Promise<StepState | null>;

package/dist/wirings/workflow/workflow-step-claim.js

@@ -0,0 +1,27 @@
+import { WorkflowStepFunctionMismatchError } from './workflow-errors.js';
+/**
+ * Decide whether this dispatch owns the step, by reading its state and then
+ * writing it — which only excludes a concurrent dispatch when the caller holds
+ * a lock that genuinely excludes one.
+ *
+ * A store that can express the whole decision as a single conditional write
+ * should do that instead of calling this.
+ */
+export const claimStepByReadThenWrite = async (store, runId, stepName, rpcName) => {
+    const stepState = await store.getStepState(runId, stepName);
+    // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
+    if (stepState.rpcName !== undefined &&
+        stepState.rpcName !== (rpcName ?? null)) {
+        throw new WorkflowStepFunctionMismatchError(runId, stepName);
+    }
+    if (stepState.status === 'succeeded' || stepState.status === 'running') {
+        return null;
+    }
+    if (stepState.status === 'failed') {
+        return store.createRetryAttempt(stepState.stepId, 'running');
+    }
+    if (stepState.status === 'pending' || stepState.status === 'scheduled') {
+        await store.setStepRunning(stepState.stepId);
+    }
+    return stepState;
+};

package/knowledge/decisions/security/a-scaffold-flag-says-a-surface-exists-not-who-may-call-it.md

@@ -0,0 +1,47 @@
+---
+type: decision
+title: A scaffold flag says a surface exists, not who may call it
+description: scaffold.<feature> is boolean | { path } — the auth field is gone, because the function runner already enforces each generated function's own auth, wiring, scopes and addon
+tags: config, authorization, codegen, scaffold
+---
+
+# A scaffold flag says a surface exists, not who may call it
+
+`scaffold.<feature>` carried an `auth` field, defaulting closed — see
+[[scaffold-features-are-authenticated-unless-opted-out]], which this supersedes.
+The field is gone. `PikkuScaffoldFeature` is now `boolean | { path?: string }`:
+whether the surface is generated, and where the file goes.
+
+The field was a second gate in front of one that already runs. `resolveSession`
+in `packages/core/src/function/function-runner.ts` throws `ForbiddenError` for a
+`pikkuFunc` with no session, unconditionally; a `pikkuSessionlessFunc` requires
+one when its own `auth: true` says so, when its wiring says so, or when
+`resolveAddonAuth` does. That is where authentication is decided, per function,
+by whoever wrote it. A config flag three directories away could only be coarser
+than that, and being coarser was the whole problem the earlier decision was
+trying to work around by defaulting it closed.
+
+So the generated dispatchers — public RPC, public agent, workflow routes and the
+events channel — emit a fixed `auth: false` on every function and wiring they
+write. That is not the scaffold declaring the surface public. It is the
+dispatcher declining to gate, because it does not know what it is forwarding to:
+`rpcCaller` hands whatever name it was given to `rpc.exposed`, and the function
+that resolves there is the one that answers for itself. Omitting the field would
+not have been neutral — a wiring with no `auth` requires a session — so the
+wrapper would reject the call before the gate that decides ever ran, which is a
+harder gate than the config field it replaced, applied by the layer least
+qualified to apply it.
+
+The scoped admin surfaces do not emit it. `userAdmin` and `virtualUser` generate
+`pikkuFunc` with `scopes: ['admin:users:...']` — session-required by
+construction, and the function that decides rather than a wrapper in front of
+one. `auth: false` there is refused by `runPikkuFunc` with a warning telling you
+to use `pikkuSessionlessFunc` instead, and `auth: true` would only restate the
+type.
+
+`pikku enable <feature>` lost its `--noAuth` flag with the field it wrote.
+
+**What this rules out:** reading a scaffold flag as an authorization decision;
+gating a surface in the config instead of on the function that answers it; and
+letting a generated dispatcher inherit the default `auth: true`, which reads as
+"undecided" and behaves as "denied".

package/knowledge/decisions/security/index.md

@@ -50,7 +50,8 @@ A rule about who may do what, and which way it fails when it is unsure.
 - [Queue job identities are signed at enqueue](queue-job-identities-are-signed-at-enqueue.md) — A job's pikkuUserId is an HMAC claim bound to the queue and payload; an unverifiable claim is dropped, never trusted
 - [Queue jobs carry the producer's pikku user id](queue-jobs-carry-the-producers-pikku-user-id.md) — A job's pikkuUserId is trusted as identity by the worker, so enqueue rights are effectively act-as-user rights
 - [Remote addon tokens are client credentials, not mesh trust](remote-addon-tokens-are-client-credentials-not-mesh-trust.md) — wireRemoteAddon authenticates as a client to a hosted library and fails closed on an empty token; it never uses PIKKU_REMOTE_SECRET
-- [A scaffolded surface is authenticated unless the config opts out in writing](scaffold-features-are-authenticated-unless-opted-out.md) — scaffold.<feature> became boolean | { auth, path }, where true means authenticated — going public requires typing { auth: false }, and the legacy 'auth' | 'no-auth' strings are refused rather than coerced
+- [A scaffold flag says a surface exists, not who may call it](a-scaffold-flag-says-a-surface-exists-not-who-may-call-it.md) — scaffold.<feature> is boolean | { path }; the auth field is gone because the function runner already enforces each generated function's own auth, wiring, scopes and addon
+- [A scaffolded surface is authenticated unless the config opts out in writing](scaffold-features-are-authenticated-unless-opted-out.md) — SUPERSEDED; retains why the legacy 'auth' | 'no-auth' strings are refused rather than coerced
 - [Scenario-step functions are never externally invocable over RPC](scenario-step-functions-are-never-externally-invocable.md) — rpcExposed requires expose and rejects scenarioStep, so test steps stay reachable only from inside a scenario run
 - [Scope resolution happens at the session boundary and scope sync never deletes](scope-resolution-happens-at-the-session-boundary-and-sync-never-deletes.md) — ScopeService is called when a session is built, never by the function runner, and syncScopes only ever adds — revoking is an explicit operation
 - [Self-authentication is declared, not detected](self-authentication-is-declared-not-detected.md) — A function that authorizes callers in its own body says so with selfAuthenticated; codegen never tries to infer it

package/knowledge/decisions/security/scaffold-features-are-authenticated-unless-opted-out.md

@@ -7,6 +7,12 @@ tags: config, authorization, codegen, scaffold
 
 # A scaffolded surface is authenticated unless the config opts out in writing
 
+**Superseded by
+[[a-scaffold-flag-says-a-surface-exists-not-who-may-call-it]]**: the `auth`
+field is gone entirely, because the function runner already enforces each
+generated function's own auth. The reasoning below still explains why the
+legacy `'auth' | 'no-auth'` strings are refused rather than coerced.
+
 `scaffold.<feature>` was `'auth' | 'no-auth' | false`. Two things were wrong
 with it, and they compounded.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.86",
+  "version": "0.12.88",
   "description": "The Pikku runtime — functions, wirings, services, middleware and types",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",

package/src/dev/hot-reload.ts

@@ -11,7 +11,7 @@ import type { Logger } from '../services/logger.js'
 import type { CorePikkuFunctionConfig } from '../function/functions.types.js'
 import { createModuleRunner } from './module-runner.js'
 
-export * from './reload-meta.js'
+export { reloadGeneratedMeta, reconcileAddonRegistry } from './reload-meta.js'
 
 interface PikkuDevReloaderOptions {
   srcDirectories: string[]

package/src/types/index.ts

@@ -1,2 +1,25 @@
-export * from './core.types.js'
+export type {
+  AuthInstance,
+  CommonWireMeta,
+  CoreConfig,
+  CoreServices,
+  CoreSingletonServices,
+  CoreUserSession,
+  CreateConfig,
+  CreateSingletonServices,
+  CreateWireServices,
+  GetCredential,
+  PikkuWire,
+  PikkuWiringTypes,
+  PostgresConfig,
+  SecretlessServices,
+  SecurityAuditIssue,
+  SecurityAuditReport,
+  SecurityAuditSummary,
+  SecurityAuditUpdate,
+  SecuritySeverity,
+  SecurityUpdateLevel,
+  ServerLifecycle,
+  WireServices,
+} from './core.types.js'
 export type * from './state.types.js'

package/src/wirings/agent/agent-memory.test.ts

@@ -74,6 +74,72 @@ describe('agent-memory', () => {
     })
   })
 
+  test('deepMergeWorkingMemory replaces an array re-emitted in full', () => {
+    const result = deepMergeWorkingMemory(
+      { steps: [{ id: 1 }, { id: 2 }], other: 'kept' },
+      { steps: [{ id: 1 }, { id: 2 }, { id: 3 }] }
+    )
+
+    assert.deepEqual(result, {
+      steps: [{ id: 1 }, { id: 2 }, { id: 3 }],
+      other: 'kept',
+    })
+  })
+
+  test('deepMergeWorkingMemory replaces rather than appends a partial array', () => {
+    const result = deepMergeWorkingMemory(
+      { steps: [{ id: 1 }, { id: 2 }] },
+      { steps: [{ id: 3 }] }
+    )
+
+    assert.deepEqual(result, { steps: [{ id: 3 }] })
+  })
+
+  test('deepMergeWorkingMemory clears an array field set to null', () => {
+    const result = deepMergeWorkingMemory(
+      { steps: [1, 2, 3], city: 'Berlin' },
+      {
+        steps: null,
+      }
+    )
+
+    assert.deepEqual(result, { city: 'Berlin' })
+  })
+
+  test('deepMergeWorkingMemory replaces arrays nested inside merged objects', () => {
+    const result = deepMergeWorkingMemory(
+      {
+        profile: {
+          name: 'Yasser',
+          tags: ['a', 'b'],
+        },
+      },
+      {
+        profile: {
+          tags: ['c'],
+        },
+      }
+    )
+
+    assert.deepEqual(result, {
+      profile: {
+        name: 'Yasser',
+        tags: ['c'],
+      },
+    })
+  })
+
+  test('deepMergeWorkingMemory replaces across array and object shape changes', () => {
+    assert.deepEqual(
+      deepMergeWorkingMemory({ value: { a: 1 } }, { value: [1, 2] }),
+      { value: [1, 2] }
+    )
+    assert.deepEqual(
+      deepMergeWorkingMemory({ value: [1, 2] }, { value: { a: 1 } }),
+      { value: { a: 1 } }
+    )
+  })
+
   test('deepMergeWorkingMemory ignores prototype-polluting keys', () => {
     const before = ({} as any).isAdmin
     deepMergeWorkingMemory(
@@ -107,6 +173,13 @@ describe('agent-memory', () => {
     assert.match(prompt, /<working_memory>/)
   })
 
+  test('buildWorkingMemoryPrompt tells the model arrays are replaced whole', () => {
+    const prompt = buildWorkingMemoryPrompt({ steps: [1] })
+
+    assert.match(prompt, /Arrays are replaced, not merged/)
+    assert.match(prompt, /repeat every item you want to keep/)
+  })
+
   test('buildWorkingMemoryPrompt marks empty memory explicitly', () => {
     const prompt = buildWorkingMemoryPrompt(null)
     assert.match(prompt, /Current working memory: \(empty\)/)

package/src/wirings/agent/agent-memory.ts

@@ -39,6 +39,17 @@ export function isWorkingMemoryEnabled(
   return !!memoryConfig?.workingMemory && !!storage
 }
 
+/**
+ * Merges a model-emitted working memory update into the stored state.
+ *
+ * Objects merge key by key, and `null` deletes a key. Arrays are replaced
+ * wholesale — deliberately, not as a side effect of the object guard below.
+ * The full state is echoed back to the model every turn by
+ * {@link buildWorkingMemoryPrompt}, so it can always re-emit an array in full,
+ * and appending instead would duplicate every item each time it did.
+ * `buildWorkingMemoryPrompt` states this contract to the model; the two must
+ * stay in agreement.
+ */
 export function deepMergeWorkingMemory(
   existing: Record<string, unknown>,
   updates: Record<string, unknown>
@@ -51,9 +62,10 @@ export function deepMergeWorkingMemory(
     const value = updates[key]
     if (value === null) {
       delete result[key]
+    } else if (Array.isArray(value)) {
+      result[key] = value
     } else if (
       typeof value === 'object' &&
-      !Array.isArray(value) &&
       typeof result[key] === 'object' &&
       result[key] !== null &&
       !Array.isArray(result[key])
@@ -100,7 +112,9 @@ export function buildWorkingMemoryPrompt(
     'When you learn new information, output a partial JSON update in <working_memory> tags. ' +
       'Only include durable facts you have actually derived or the user has confirmed. ' +
       'Do not output templates, placeholders, or narration. ' +
-      'Only include changed fields. Leave unknown fields untouched. Set a field to null to delete it.'
+      'Only include changed fields. Leave unknown fields untouched. Set a field to null to delete it. ' +
+      'Arrays are replaced, not merged: to change one, repeat every item you want to keep alongside the new ones, ' +
+      'because any item you leave out is deleted.'
   )
 
   return parts.join('\n\n')

package/src/wirings/agent/agent-middleware.types.test.ts

@@ -0,0 +1,41 @@
+// Type-checked via tsconfig.type-tests.json; no runtime assertions.
+
+import { pikkuAgentMiddleware } from '../../middleware/middleware-factories.js'
+import type { PikkuAgentMiddlewareHooks } from './agent.types.js'
+
+const _singletonServicesCompile = pikkuAgentMiddleware<{ count: number }>({
+  modifyInput: async ({ logger }, { messages, instructions }) => {
+    logger.info(`agent input: ${messages.length} messages`)
+    return { messages, instructions }
+  },
+  modifyOutputStream: async ({ logger }, { event, state }) => {
+    state.count = (state.count ?? 0) + 1
+    logger.info(`event: ${event.type}`)
+    return event
+  },
+  afterStep: async ({ variables }, { stepNumber }) => {
+    variables.get(`STEP_${stepNumber}`)
+  },
+  onError: async ({ logger }, { error }) => {
+    logger.error(error.message)
+  },
+})
+void _singletonServicesCompile
+
+const _wireServiceIsNotAvailable: PikkuAgentMiddlewareHooks = {
+  // @ts-expect-error — `http` is a wire service, and an agent run has no wire
+  modifyInput: async ({ http }, { messages, instructions }) => {
+    void http
+    return { messages, instructions }
+  },
+}
+void _wireServiceIsNotAvailable
+
+const _wireServiceIsNotAvailableToTheFactory = pikkuAgentMiddleware({
+  // @ts-expect-error — `channel` is a wire service, and an agent run has no wire
+  afterToolCall: async ({ channel }, { result }) => {
+    void channel
+    return { result }
+  },
+})
+void _wireServiceIsNotAvailableToTheFactory