@pikku/core 0.12.84 → 0.12.86

package/src/wirings/secret/validate-secret-definitions.ts

@@ -54,6 +54,7 @@ export function validateAndBuildSecretDefinitionsMeta(
           description: def.description,
           secretId: def.secretId,
           schema: def.schema,
+          optional: def.optional,
           oauth2: def.oauth2,
           rotationPeriod: def.rotationPeriod,
           allowedHosts: def.allowedHosts,
@@ -73,6 +74,7 @@ export function validateAndBuildSecretDefinitionsMeta(
         description: def.description,
         secretId: def.secretId,
         schema: def.schema,
+        optional: def.optional,
         oauth2: def.oauth2,
         rotationPeriod: def.rotationPeriod,
         allowedHosts: def.allowedHosts,

package/src/wirings/variable/validate-variable-definitions.test.ts

@@ -88,4 +88,27 @@ describe('validateAndBuildVariableDefinitionsMeta', () => {
     const result = validateAndBuildVariableDefinitionsMeta([], new Map())
     assert.deepStrictEqual(result, {})
   })
+
+  test('carries required into the meta so only the opted-in ones block a deploy', () => {
+    const result = validateAndBuildVariableDefinitionsMeta(
+      [
+        {
+          name: 'consoleUrl',
+          displayName: 'Console URL',
+          variableId: 'CONSOLE_URL',
+          required: true,
+          sourceFile: 'a.ts',
+        },
+        {
+          name: 'corsOrigins',
+          displayName: 'Allowed Browser Origins',
+          variableId: 'CORS_ORIGINS',
+          sourceFile: 'a.ts',
+        },
+      ] as any,
+      new Map()
+    )
+    assert.strictEqual(result['consoleUrl']!.required, true)
+    assert.strictEqual(result['corsOrigins']!.required, undefined)
+  })
 })

package/src/wirings/variable/validate-variable-definitions.ts

@@ -43,6 +43,7 @@ export function validateAndBuildVariableDefinitionsMeta(
           description: def.description,
           variableId: def.variableId,
           schema: def.schema,
+          required: def.required,
           docsUrl: def.docsUrl,
           sourceFile: def.sourceFile,
         }
@@ -59,6 +60,7 @@ export function validateAndBuildVariableDefinitionsMeta(
         description: def.description,
         variableId: def.variableId,
         schema: def.schema,
+        required: def.required,
         docsUrl: def.docsUrl,
         sourceFile: def.sourceFile,
       }

package/src/wirings/variable/variable.types.ts

@@ -4,6 +4,14 @@ export type CoreVariable<T = unknown> = {
   description?: string
   variableId: string
   schema: T
+  /**
+   * A variable is OPTIONAL by default because `variables.get` returns
+   * `T | undefined` and never throws — every caller already handles absence, so
+   * a deploy gate that blocks on one contradicts the API. Mark a variable
+   * `required` for the few whose absence genuinely breaks the app; only those
+   * block a deploy.
+   */
+  required?: boolean
   docsUrl?: string
 }
 
@@ -13,6 +21,7 @@ export type VariableDefinitionMeta = {
   description?: string
   variableId: string
   schema?: Record<string, unknown> | string
+  required?: boolean
   docsUrl?: string
   sourceFile?: string
 }

package/src/wirings/virtual-user/index.ts

@@ -57,10 +57,7 @@ export {
   type AgentReachability,
   type ReachableAgent,
 } from './virtual-user-agents.js'
-export {
-  IntentStack,
-  intentsForPersona,
-} from './virtual-user-intents.js'
+export { IntentStack, intentsForPersona } from './virtual-user-intents.js'
 export {
   deriveCatalogue,
   deriveIntents,

package/src/wirings/virtual-user/prepare-virtual-user-run.ts

@@ -1,4 +1,4 @@
-import type { FunctionsMeta } from '../../types/core.types.js'
+import type { FunctionsMeta } from '../../function/function-meta.types.js'
 import type { WorkflowsMeta } from '../workflow/workflow.types.js'
 import { flattenSystemRoleDefinitions } from '../role/validate-role-definitions.js'
 import type {

package/src/wirings/virtual-user/virtual-user-derive.test.ts

@@ -3,7 +3,7 @@ import assert from 'node:assert/strict'
 
 import { deriveCatalogue, deriveIntents } from './virtual-user-derive.js'
 import { personaVirtualUserTarget } from './virtual-user-target.js'
-import type { FunctionsMeta } from '../../types/core.types.js'
+import type { FunctionsMeta } from '../../function/function-meta.types.js'
 import type { WorkflowsMeta } from '../workflow/workflow.types.js'
 import type {
   ScenarioPersona,

package/src/wirings/virtual-user/virtual-user-derive.ts

@@ -1,4 +1,4 @@
-import type { FunctionsMeta } from '../../types/core.types.js'
+import type { FunctionsMeta } from '../../function/function-meta.types.js'
 import type { WorkflowsMeta } from '../workflow/workflow.types.js'
 import type { ScenarioStepPhase } from '../workflow/scenario-step.types.js'
 import { composeStepProse } from '../workflow/scenario-prose.js'

package/src/wirings/virtual-user/virtual-user.types.ts

@@ -166,11 +166,7 @@ export type VirtualUserAction =
 
 /** How an intent ended, or that it has not. */
 export type IntentStatus =
-  | 'open'
-  | 'suspended'
-  | 'completed'
-  | 'abandoned'
-  | 'stuck'
+  'open' | 'suspended' | 'completed' | 'abandoned' | 'stuck'
 
 /** One intent's life within a run. */
 export interface IntentRecord {

package/src/wirings/workflow/graph/graph-foreach-queued.test.ts

@@ -0,0 +1,569 @@
+/**
+ * forEach fan-out over the durable (queued) path.
+ *
+ * `graph-foreach.test.ts` drives the inline runner, which plans and executes in
+ * one process. Production runs instead go out to a queue: entry nodes are
+ * queued by `runWorkflowGraph`, each step comes back through
+ * `executeWorkflowStep` (which has to map `node[i]` back to `node` before it
+ * can find the RPC), and every completion re-enters `continueGraph` through the
+ * orchestrator. These tests exercise that loop.
+ */
+
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../../services/in-memory-workflow-service.js'
+import { runWorkflowGraph } from './graph-runner.js'
+import { pikkuState } from '../../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+interface QueuedJob {
+  queueName: string
+  data: any
+}
+
+type RpcImpl = (rpcName: string, data: any) => Promise<any>
+
+interface Harness {
+  ws: InMemoryWorkflowService
+  rpc: any
+  jobs: QueuedJob[]
+  onEnqueue: (fn: (job: QueuedJob) => void | Promise<void>) => void
+  drain: (max?: number) => Promise<void>
+}
+
+const isOrchestratorJob = (queueName: string) =>
+  queueName.includes('orchestrator')
+
+function harness(rpcImpl: RpcImpl): Harness {
+  const jobs: QueuedJob[] = []
+  let enqueueHook: ((job: QueuedJob) => void | Promise<void>) | undefined
+
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+    queueService: {
+      add: async (queueName: string, data: any) => {
+        const job = { queueName, data }
+        jobs.push(job)
+        await enqueueHook?.(job)
+        return 'job-1'
+      },
+    },
+  } as any)
+
+  const ws = new InMemoryWorkflowService()
+  const rpc = { rpcWithWire: rpcImpl } as any
+
+  const drain = async (max = 200) => {
+    let processed = 0
+    while (jobs.length) {
+      if (++processed > max) {
+        throw new Error('the queue never settled — the fan-out is looping')
+      }
+      const job = jobs.shift()!
+      if (isOrchestratorJob(job.queueName)) {
+        await ws.orchestrateWorkflow(job.data.runId, rpc)
+      } else {
+        await ws.executeWorkflowStep(
+          job.data.runId,
+          job.data.stepName,
+          job.data.rpcName,
+          job.data.data,
+          rpc
+        )
+      }
+    }
+  }
+
+  return {
+    ws,
+    rpc,
+    jobs,
+    onEnqueue: (fn) => {
+      enqueueHook = fn
+    },
+    drain,
+  }
+}
+
+function seedMeta(name: string, nodes: Record<string, any>, entry: string[]) {
+  const metaState = pikkuState(null, 'workflows', 'meta')
+  metaState[name] = {
+    name,
+    pikkuFuncId: name,
+    source: 'graph',
+    entryNodeIds: entry,
+    graphHash: `${name}-hash`,
+    nodes,
+  }
+  return () => {
+    delete metaState[name]
+  }
+}
+
+const stepNames = async (ws: InMemoryWorkflowService, runId: string) =>
+  (await ws.getStepInstances(runId)).map((i) => i.stepName).sort()
+
+describe('graph forEach fanout — durable queued path', () => {
+  test('every item gets its own queued step, and each one resolves $item', async () => {
+    const seen: any[] = []
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows')
+        return { rows: [{ n: 1 }, { n: 2 }, { n: 3 }] }
+      if (rpcName === 'handleRow') {
+        seen.push(data)
+        return { doubled: data.value * 2 }
+      }
+      return {}
+    })
+    const cleanup = seedMeta(
+      'queuedFanout',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list', path: 'rows' },
+          input: { value: { $ref: '$item', path: 'n' } },
+        },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanout',
+      {},
+      h.rpc,
+      false
+    )
+    await h.drain()
+
+    assert.equal((await h.ws.getRun(runId))?.status, 'completed')
+    assert.deepEqual(await stepNames(h.ws, runId), [
+      'handle[0]',
+      'handle[1]',
+      'handle[2]',
+      'list',
+    ])
+    assert.deepEqual(
+      seen.map((d) => d.value).sort(),
+      [1, 2, 3],
+      `each queued instance should carry its own item, got ${JSON.stringify(seen)}`
+    )
+
+    cleanup()
+  })
+
+  test('a parallel fanout queues every item in one orchestrator pass', async () => {
+    const h = harness(async (rpcName) => {
+      if (rpcName === 'listRows') return [1, 2, 3]
+      return { ok: true }
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutBurst',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list' },
+          input: { value: { $ref: '$item' } },
+        },
+      },
+      ['list']
+    )
+
+    await runWorkflowGraph(h.ws, 'queuedFanoutBurst', {}, h.rpc, false)
+
+    // list, then the orchestrator pass that fans handle out.
+    while (h.jobs.length) {
+      const job = h.jobs.shift()!
+      if (isOrchestratorJob(job.queueName)) {
+        await h.ws.orchestrateWorkflow(job.data.runId, h.rpc)
+        break
+      }
+      await h.ws.executeWorkflowStep(
+        job.data.runId,
+        job.data.stepName,
+        job.data.rpcName,
+        job.data.data,
+        h.rpc
+      )
+    }
+
+    const queuedSteps = h.jobs
+      .filter((j) => !isOrchestratorJob(j.queueName))
+      .map((j) => j.data.stepName)
+    assert.deepEqual(
+      queuedSteps,
+      ['handle[0]', 'handle[1]', 'handle[2]'],
+      'a parallel fanout should have all of its items in flight at once'
+    )
+
+    cleanup()
+  })
+
+  test("mode: 'sequential' never has two items queued at once", async () => {
+    const executed: string[] = []
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows') return ['a', 'b', 'c']
+      executed.push(data.value)
+      return { value: data.value }
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutSequential',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          mode: 'sequential',
+          forEach: { $ref: 'list' },
+          input: { value: { $ref: '$item' } },
+        },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanoutSequential',
+      {},
+      h.rpc,
+      false
+    )
+
+    h.onEnqueue(async (job) => {
+      if (isOrchestratorJob(job.queueName)) return
+      if (!job.data.stepName.startsWith('handle[')) return
+      const unfinished = (await h.ws.getStepInstances(job.data.runId)).filter(
+        (i) =>
+          i.stepName.startsWith('handle[') &&
+          i.stepName !== job.data.stepName &&
+          i.status !== 'succeeded'
+      )
+      assert.deepEqual(
+        unfinished.map((i) => i.stepName),
+        [],
+        `${job.data.stepName} was queued while earlier items were still unfinished`
+      )
+    })
+
+    await h.drain()
+
+    assert.equal((await h.ws.getRun(runId))?.status, 'completed')
+    assert.deepEqual(
+      executed,
+      ['a', 'b', 'c'],
+      'sequential items should run in source order'
+    )
+
+    cleanup()
+  })
+
+  test('a downstream node fires once, after every item, with the ordered results', async () => {
+    const downstreamCalls: any[] = []
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows') return [1, 2, 3]
+      if (rpcName === 'handleRow') return { squared: data.value * data.value }
+      if (rpcName === 'summarise') {
+        downstreamCalls.push(data)
+        return { ok: true }
+      }
+      return {}
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutAggregate',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list' },
+          input: { value: { $ref: '$item' } },
+          next: 'summary',
+        },
+        summary: {
+          nodeId: 'summary',
+          rpcName: 'summarise',
+          input: { results: { $ref: 'handle' } },
+        },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanoutAggregate',
+      {},
+      h.rpc,
+      false
+    )
+    await h.drain()
+
+    assert.equal((await h.ws.getRun(runId))?.status, 'completed')
+    assert.equal(
+      downstreamCalls.length,
+      1,
+      `the downstream node should run once, not once per item (ran ${downstreamCalls.length}x)`
+    )
+    assert.deepEqual(downstreamCalls[0], {
+      results: [{ squared: 1 }, { squared: 4 }, { squared: 9 }],
+    })
+
+    cleanup()
+  })
+
+  test('a downstream node stays put while sibling items are still in flight', async () => {
+    // The orchestrator job for item 0 lands while items 1 and 2 are still
+    // sitting on the step queue — only a completion count that ignores the
+    // unfinished siblings would release the downstream node here.
+    const downstreamCalls: any[] = []
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows') return [1, 2, 3]
+      if (rpcName === 'handleRow') return { squared: data.value * data.value }
+      if (rpcName === 'summarise') {
+        downstreamCalls.push(data)
+        return { ok: true }
+      }
+      return {}
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutEagerOrchestrator',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list' },
+          input: { value: { $ref: '$item' } },
+          next: 'summary',
+        },
+        summary: {
+          nodeId: 'summary',
+          rpcName: 'summarise',
+          input: { results: { $ref: 'handle' } },
+        },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanoutEagerOrchestrator',
+      {},
+      h.rpc,
+      false
+    )
+
+    // An orchestrator job runs as soon as it exists, rather than behind the
+    // items its siblings are still waiting on.
+    let processed = 0
+    while (h.jobs.length) {
+      if (++processed > 200) throw new Error('the queue never settled')
+      const index = h.jobs.findIndex((j) => isOrchestratorJob(j.queueName))
+      const job = h.jobs.splice(index === -1 ? 0 : index, 1)[0]!
+      if (isOrchestratorJob(job.queueName)) {
+        const unfinishedItems = (await h.ws.getStepInstances(runId)).filter(
+          (i) => i.stepName.startsWith('handle[') && i.status !== 'succeeded'
+        )
+        await h.ws.orchestrateWorkflow(job.data.runId, h.rpc)
+        if (unfinishedItems.length > 0) {
+          assert.deepEqual(
+            downstreamCalls,
+            [],
+            `the downstream node ran while ${unfinishedItems.length} item(s) were unfinished`
+          )
+        }
+      } else {
+        await h.ws.executeWorkflowStep(
+          job.data.runId,
+          job.data.stepName,
+          job.data.rpcName,
+          job.data.data,
+          h.rpc
+        )
+      }
+    }
+
+    assert.equal((await h.ws.getRun(runId))?.status, 'completed')
+    assert.deepEqual(downstreamCalls, [
+      { results: [{ squared: 1 }, { squared: 4 }, { squared: 9 }] },
+    ])
+
+    cleanup()
+  })
+
+  test('an entry node fans out over the trigger input', async () => {
+    const seen: any[] = []
+    const h = harness(async (_rpcName, data) => {
+      seen.push(data)
+      return { ok: true }
+    })
+    const cleanup = seedMeta(
+      'queuedEntryFanout',
+      {
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'trigger', path: 'ids' },
+          input: { id: { $ref: '$item' } },
+        },
+      },
+      ['handle']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedEntryFanout',
+      { ids: ['x', 'y'] },
+      h.rpc,
+      false
+    )
+    await h.drain()
+
+    assert.equal((await h.ws.getRun(runId))?.status, 'completed')
+    assert.deepEqual(await stepNames(h.ws, runId), ['handle[0]', 'handle[1]'])
+    assert.deepEqual(seen.map((d) => d.id).sort(), ['x', 'y'])
+
+    cleanup()
+  })
+
+  test('a sequential entry fanout still runs every item', async () => {
+    const executed: string[] = []
+    const h = harness(async (_rpcName, data) => {
+      executed.push(data.id)
+      return { ok: true }
+    })
+    const cleanup = seedMeta(
+      'queuedEntryFanoutSequential',
+      {
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          mode: 'sequential',
+          forEach: { $ref: 'trigger', path: 'ids' },
+          input: { id: { $ref: '$item' } },
+        },
+      },
+      ['handle']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedEntryFanoutSequential',
+      { ids: ['x', 'y', 'z'] },
+      h.rpc,
+      false
+    )
+    await h.drain()
+
+    assert.equal(
+      (await h.ws.getRun(runId))?.status,
+      'completed',
+      'an entry fanout that starts one item at a time must still reach the rest'
+    )
+    assert.deepEqual(executed, ['x', 'y', 'z'])
+
+    cleanup()
+  })
+
+  test('a failing item fails the run instead of stalling it', async () => {
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows') return [1, 2, 3]
+      if (data.value === 2) throw new Error('item 2 exploded')
+      return { ok: true }
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutFailure',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list' },
+          retries: 0,
+          input: { value: { $ref: '$item' } },
+          next: 'summary',
+        },
+        summary: { nodeId: 'summary', rpcName: 'summarise' },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanoutFailure',
+      {},
+      h.rpc,
+      false
+    )
+    await h.drain().catch(() => {})
+
+    const run = await h.ws.getRun(runId)
+    assert.notEqual(
+      run?.status,
+      'completed',
+      'a run with a failed item must never report success'
+    )
+    const instances = await h.ws.getStepInstances(runId)
+    assert.equal(
+      instances.some((i) => i.stepName === 'summary'),
+      false,
+      'the downstream node must not run when an item failed'
+    )
+
+    cleanup()
+  })
+
+  test('re-orchestrating a settled fanout does not re-queue its items', async () => {
+    const calls: any[] = []
+    const h = harness(async (rpcName, data) => {
+      if (rpcName === 'listRows') return [1, 2]
+      calls.push(data)
+      return { ok: true }
+    })
+    const cleanup = seedMeta(
+      'queuedFanoutIdempotent',
+      {
+        list: { nodeId: 'list', rpcName: 'listRows', next: 'handle' },
+        handle: {
+          nodeId: 'handle',
+          rpcName: 'handleRow',
+          forEach: { $ref: 'list' },
+          input: { value: { $ref: '$item' } },
+        },
+      },
+      ['list']
+    )
+
+    const { runId } = await runWorkflowGraph(
+      h.ws,
+      'queuedFanoutIdempotent',
+      {},
+      h.rpc,
+      false
+    )
+    await h.drain()
+    assert.equal(calls.length, 2)
+
+    // A duplicated orchestrator job — queues redeliver.
+    await h.ws.orchestrateWorkflow(runId, h.rpc)
+    await h.drain()
+
+    assert.equal(
+      calls.length,
+      2,
+      `a redelivered orchestrator job re-ran the fanout (${calls.length} calls)`
+    )
+    assert.deepEqual(await stepNames(h.ws, runId), [
+      'handle[0]',
+      'handle[1]',
+      'list',
+    ])
+
+    cleanup()
+  })
+})