@pikku/core 0.11.2 → 0.12.0

package/src/wirings/variable/validate-variable-definitions.ts

@@ -0,0 +1,69 @@
+import type {
+  VariableDefinitions,
+  VariableDefinitionsMeta,
+} from './variable.types.js'
+
+export interface SchemaRefLike {
+  variableName: string
+  sourceFile: string
+}
+
+export function validateAndBuildVariableDefinitionsMeta(
+  definitions: VariableDefinitions,
+  schemaLookup: Map<string, SchemaRefLike>
+): VariableDefinitionsMeta {
+  const meta: VariableDefinitionsMeta = {}
+  const variableIdToDefinition: Map<string, VariableDefinitions[0]> = new Map()
+
+  for (const def of definitions) {
+    const existingDef = variableIdToDefinition.get(def.variableId)
+
+    if (existingDef) {
+      if (def.schema && existingDef.schema) {
+        const defSchemaRef = schemaLookup.get(def.schema as string)
+        const existingSchemaRef = schemaLookup.get(existingDef.schema as string)
+
+        if (defSchemaRef && existingSchemaRef) {
+          if (
+            defSchemaRef.variableName !== existingSchemaRef.variableName ||
+            defSchemaRef.sourceFile !== existingSchemaRef.sourceFile
+          ) {
+            throw new Error(
+              `Variable '${def.variableId}' is defined with different schemas.\n` +
+                `  First definition: ${existingDef.sourceFile} (schema: ${existingSchemaRef.variableName})\n` +
+                `  Second definition: ${def.sourceFile} (schema: ${defSchemaRef.variableName})\n` +
+                `Variables sharing a variableId must use the same schema.`
+            )
+          }
+        }
+      }
+
+      if (!meta[def.name]) {
+        meta[def.name] = {
+          name: def.name,
+          displayName: def.displayName,
+          description: def.description,
+          variableId: def.variableId,
+          schema: def.schema,
+          sourceFile: def.sourceFile,
+        }
+      }
+      continue
+    }
+
+    variableIdToDefinition.set(def.variableId, def)
+
+    if (!meta[def.name]) {
+      meta[def.name] = {
+        name: def.name,
+        displayName: def.displayName,
+        description: def.description,
+        variableId: def.variableId,
+        schema: def.schema,
+        sourceFile: def.sourceFile,
+      }
+    }
+  }
+
+  return meta
+}

package/src/wirings/variable/variable.types.ts

@@ -0,0 +1,22 @@
+export type CoreVariable<T = unknown> = {
+  name: string
+  displayName: string
+  description?: string
+  variableId: string
+  schema: T
+}
+
+export type VariableDefinitionMeta = {
+  name: string
+  displayName: string
+  description?: string
+  variableId: string
+  schema?: Record<string, unknown> | string
+  sourceFile?: string
+}
+
+export type VariableDefinitionsMeta = Record<string, VariableDefinitionMeta>
+
+export type VariableDefinitions = VariableDefinitionMeta[]
+
+export const wireVariable = <T>(_config: CoreVariable<T>): void => {}

package/src/wirings/workflow/dsl/index.ts

@@ -8,6 +8,7 @@ export type {
   WorkflowWireDoRPC,
   WorkflowWireDoInline,
   WorkflowWireSleep,
+  WorkflowWireSuspend,
   InputSource,
   OutputBinding,
   RpcStepMeta,

package/src/wirings/workflow/dsl/workflow-dsl.types.ts

@@ -45,6 +45,11 @@ export type WorkflowWireSleep = (
   duration: string
 ) => Promise<void>
 
+/**
+ * Type signature for workflow.suspend() - used by inspector
+ */
+export type WorkflowWireSuspend = (reason: string) => Promise<void>
+
 /**
  * Input source for step arguments in DSL workflows
  */
@@ -298,7 +303,7 @@ export interface WorkflowStepWire {
  */
 export interface PikkuWorkflowWire {
   /** The workflow name */
-  workflowName: string
+  name: string
   /** The current run ID */
   runId: string
   /** Get the current workflow run */
@@ -309,4 +314,7 @@ export interface PikkuWorkflowWire {
 
   /** Sleep for a duration */
   sleep: WorkflowWireSleep
+
+  /** Suspend workflow until explicitly resumed */
+  suspend: WorkflowWireSuspend
 }

package/src/wirings/workflow/dsl/workflow-runner.ts

@@ -1,5 +1,6 @@
 import { pikkuState } from '../../../pikku-state.js'
 import { addFunction } from '../../../function/function-runner.js'
+import { PikkuMissingMetaError } from '../../../errors/errors.js'
 
 /**
  * Add a workflow to the system
@@ -10,8 +11,8 @@ export const addWorkflow = (workflowName: string, workflowFunc: any) => {
   const meta = pikkuState(null, 'workflows', 'meta')
   const workflowMeta = meta[workflowName]
   if (!workflowMeta) {
-    throw new Error(
-      `Workflow metadata not found for '${workflowName}'. Make sure to run the CLI to generate metadata.`
+    throw new PikkuMissingMetaError(
+      `Missing generated metadata for workflow '${workflowName}'`
     )
   }
 
@@ -23,5 +24,5 @@ export const addWorkflow = (workflowName: string, workflowFunc: any) => {
   })
 
   // Register the function with pikku
-  addFunction(workflowMeta.pikkuFuncName, workflowFunc)
+  addFunction(workflowMeta.pikkuFuncId, workflowFunc)
 }

package/src/wirings/workflow/graph/graph-node.ts

@@ -54,7 +54,7 @@ export interface GraphNodeDef<
  * @example
  * ```typescript
  * // Import the typed graph from generated types
- * import { pikkuWorkflowGraph, wireWorkflow } from './.pikku/workflow/pikku-workflow-types.gen.js'
+ * import { pikkuWorkflowGraph } from './.pikku/workflow/pikku-workflow-types.gen.js'
  *
  * export const myGraph = pikkuWorkflowGraph({
  *   name: 'myWorkflow',
@@ -71,11 +71,6 @@ export interface GraphNodeDef<
  *     },
  *   },
  * })
- *
- * wireWorkflow({
- *   wires: { http: { route: '/start', method: 'post' } },
- *   graph: myGraph,
- * })
  * ```
  */
 /**

package/src/wirings/workflow/graph/graph-runner.test.ts

@@ -0,0 +1,495 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../../services/in-memory-workflow-service.js'
+import {
+  continueGraph,
+  executeGraphStep,
+  runWorkflowGraph,
+} from './graph-runner.js'
+import type { WorkflowRuntimeMeta } from '../workflow.types.js'
+import { pikkuState } from '../../../pikku-state.js'
+import { RPCNotFoundError } from '../../rpc/rpc-runner.js'
+
+describe('graph-runner bugs', () => {
+  test('continueGraph should NOT mark workflow completed while nodes are still running', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const meta: WorkflowRuntimeMeta = {
+      name: 'testPrematureCompletion',
+      pikkuFuncId: 'testPrematureCompletion',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'test-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', next: 'b' },
+        b: { nodeId: 'b', rpcName: 'doB' },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testPrematureCompletion',
+      {},
+      false,
+      'test-hash',
+      { type: 'test' }
+    )
+
+    const stepA = await ws.insertStepState(runId, 'a', 'doA', {})
+    await ws.setStepRunning(stepA.stepId)
+    await ws.setStepResult(stepA.stepId, { ok: true })
+
+    const stepB = await ws.insertStepState(runId, 'b', 'doB', {})
+    await ws.setStepRunning(stepB.stepId)
+
+    await continueGraph(ws, runId, 'testPrematureCompletion', meta)
+
+    const run = await ws.getRun(runId)
+    assert.notEqual(
+      run?.status,
+      'completed',
+      'workflow should NOT be completed while node b is still running'
+    )
+    assert.equal(run?.status, 'running')
+  })
+
+  test('runWorkflowGraph should throw and not create a run for unknown input refs', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testOrphanedRun'] = {
+      name: 'testOrphanedRun',
+      pikkuFuncId: 'testOrphanedRun',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'orphan-hash',
+      nodes: {
+        a: {
+          nodeId: 'a',
+          rpcName: 'doA',
+          input: { dep: { $ref: 'someOtherNode', path: 'value' } },
+        },
+      },
+    }
+
+    await assert.rejects(() => runWorkflowGraph(ws, 'testOrphanedRun', {}), {
+      message: /references unknown node 'someOtherNode' in input/,
+    })
+
+    const runs: string[] = [...(ws as any).runs.keys()]
+    assert.equal(
+      runs.length,
+      0,
+      'no run should be created when entry nodes have unsatisfied dependencies'
+    )
+
+    delete metaState['testOrphanedRun']
+  })
+
+  test('runWorkflowGraph should throw and not create a run for unknown next targets', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testUnknownNextTarget'] = {
+      name: 'testUnknownNextTarget',
+      pikkuFuncId: 'testUnknownNextTarget',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'unknown-next-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', next: 'missingNode' },
+      },
+    }
+
+    await assert.rejects(
+      () => runWorkflowGraph(ws, 'testUnknownNextTarget', {}),
+      {
+        message: /routes to unknown node 'missingNode'/,
+      }
+    )
+
+    const runs: string[] = [...(ws as any).runs.keys()]
+    assert.equal(
+      runs.length,
+      0,
+      'no run should be created for invalid next targets'
+    )
+
+    delete metaState['testUnknownNextTarget']
+  })
+
+  test('continueGraph should throw for ambiguous template step remapping', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const meta: WorkflowRuntimeMeta = {
+      name: 'testAmbiguousTemplateStepRemap',
+      pikkuFuncId: 'testAmbiguousTemplateStepRemap',
+      source: 'graph',
+      entryNodeIds: ['start'],
+      graphHash: 'ambiguous-step-remap-hash',
+      nodes: {
+        start: { nodeId: 'start', rpcName: 'doStart' },
+        'task-${id}': { nodeId: 'task-${id}', rpcName: 'doTaskById' },
+        'task-${name}': { nodeId: 'task-${name}', rpcName: 'doTaskByName' },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testAmbiguousTemplateStepRemap',
+      {},
+      false,
+      'ambiguous-step-remap-hash',
+      { type: 'test' }
+    )
+
+    const step = await ws.insertStepState(runId, 'task-123', 'doTask', {})
+    await ws.setStepRunning(step.stepId)
+    await ws.setStepResult(step.stepId, { ok: true })
+
+    await assert.rejects(
+      () => continueGraph(ws, runId, 'testAmbiguousTemplateStepRemap', meta),
+      {
+        message: /ambiguous template node match for 'task-123'/i,
+      }
+    )
+  })
+
+  test('continueGraph should throw for ambiguous template branch key remapping', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const meta: WorkflowRuntimeMeta = {
+      name: 'testAmbiguousTemplateBranchRemap',
+      pikkuFuncId: 'testAmbiguousTemplateBranchRemap',
+      source: 'graph',
+      entryNodeIds: ['start'],
+      graphHash: 'ambiguous-branch-remap-hash',
+      nodes: {
+        start: { nodeId: 'start', rpcName: 'doStart' },
+        'branch-${id}': { nodeId: 'branch-${id}', rpcName: 'doBranchById' },
+        'branch-${name}': {
+          nodeId: 'branch-${name}',
+          rpcName: 'doBranchByName',
+        },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testAmbiguousTemplateBranchRemap',
+      {},
+      false,
+      'ambiguous-branch-remap-hash',
+      { type: 'test' }
+    )
+
+    const step = await ws.insertStepState(runId, 'branch-1', 'doBranch', {})
+    await ws.setBranchTaken(step.stepId, 'next')
+
+    await assert.rejects(
+      () => continueGraph(ws, runId, 'testAmbiguousTemplateBranchRemap', meta),
+      {
+        message: /ambiguous template branch key match for 'branch-1'/i,
+      }
+    )
+  })
+
+  test('continueGraph should queue converging next node only once', async () => {
+    const ws = new InMemoryWorkflowService()
+    const queuedNodes: string[] = []
+
+    ws.setServices(
+      {
+        queueService: {
+          add: async (_queueName: string, data: any) => {
+            if (data?.stepName) {
+              queuedNodes.push(data.stepName)
+            }
+          },
+        },
+      } as any,
+      (() => ({})) as any,
+      {} as any
+    )
+
+    const meta: WorkflowRuntimeMeta = {
+      name: 'testConvergingNextNode',
+      pikkuFuncId: 'testConvergingNextNode',
+      source: 'graph',
+      entryNodeIds: ['a', 'b'],
+      graphHash: 'converging-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', next: 'c' },
+        b: { nodeId: 'b', rpcName: 'doB', next: 'c' },
+        c: { nodeId: 'c', rpcName: 'doC' },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testConvergingNextNode',
+      {},
+      false,
+      'converging-hash',
+      { type: 'test' }
+    )
+
+    const stepA = await ws.insertStepState(runId, 'a', 'doA', {})
+    await ws.setStepRunning(stepA.stepId)
+    await ws.setStepResult(stepA.stepId, { ok: true })
+
+    const stepB = await ws.insertStepState(runId, 'b', 'doB', {})
+    await ws.setStepRunning(stepB.stepId)
+    await ws.setStepResult(stepB.stepId, { ok: true })
+
+    await continueGraph(ws, runId, 'testConvergingNextNode', meta)
+
+    assert.deepEqual(queuedNodes, ['c'])
+  })
+
+  test('inline graph should execute converging next node only once', async () => {
+    const ws = new InMemoryWorkflowService()
+    let cCalls = 0
+
+    const mockRpcService = {
+      rpcWithWire: async (rpcName: string) => {
+        if (rpcName === 'doA' || rpcName === 'doB') {
+          return { ok: true }
+        }
+        if (rpcName === 'doC') {
+          cCalls += 1
+          return { ok: true }
+        }
+        return {}
+      },
+    }
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testInlineConvergingSingleExecution'] = {
+      name: 'testInlineConvergingSingleExecution',
+      pikkuFuncId: 'testInlineConvergingSingleExecution',
+      source: 'graph',
+      entryNodeIds: ['a', 'b'],
+      graphHash: 'inline-converging-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', next: 'c' },
+        b: { nodeId: 'b', rpcName: 'doB', next: 'c' },
+        c: { nodeId: 'c', rpcName: 'doC' },
+      },
+    }
+
+    await runWorkflowGraph(
+      ws,
+      'testInlineConvergingSingleExecution',
+      {},
+      mockRpcService,
+      true
+    )
+
+    assert.equal(cCalls, 1)
+
+    delete metaState['testInlineConvergingSingleExecution']
+  })
+
+  test('inline graph node failure should mark workflow as failed', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const mockRpcService = {
+      rpcWithWire: async (rpcName: string, _data: any, _wire: any) => {
+        if (rpcName === 'doA') return { ok: true }
+        if (rpcName === 'doB') throw new Error('doB exploded')
+        return {}
+      },
+    }
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testInlineFailure'] = {
+      name: 'testInlineFailure',
+      pikkuFuncId: 'testInlineFailure',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'inline-fail-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', next: 'b' },
+        b: { nodeId: 'b', rpcName: 'doB' },
+      },
+    }
+
+    let runId: string | undefined
+    try {
+      const result = await runWorkflowGraph(
+        ws,
+        'testInlineFailure',
+        {},
+        mockRpcService,
+        true
+      )
+      runId = result.runId
+    } catch (_) {
+      const runs = (ws as any).runs as Map<string, any>
+      for (const [id] of runs) {
+        runId = id
+        break
+      }
+    }
+
+    assert.ok(runId, 'should have created a run')
+    const run = await ws.getRun(runId!)
+    assert.equal(
+      run?.status,
+      'failed',
+      'workflow should be failed after inline node error, not left running'
+    )
+
+    delete metaState['testInlineFailure']
+  })
+
+  test('executeGraphStep should throw after queueing onError nodes', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    ws.setServices(
+      {
+        queueService: {
+          add: async () => {},
+        },
+      } as any,
+      (() => ({})) as any,
+      {} as any
+    )
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testQueuedOnErrorThrow'] = {
+      name: 'testQueuedOnErrorThrow',
+      pikkuFuncId: 'testQueuedOnErrorThrow',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'queued-on-error-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'doA', onError: 'onErr' },
+        onErr: { nodeId: 'onErr', rpcName: 'handleErr' },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testQueuedOnErrorThrow',
+      {},
+      false,
+      'queued-on-error-hash',
+      { type: 'test' }
+    )
+    const step = await ws.insertStepState(runId, 'a', 'doA', {})
+
+    const rpcService = {
+      rpcWithWire: async () => {
+        throw new Error('boom')
+      },
+    }
+
+    await assert.rejects(
+      () =>
+        executeGraphStep(
+          ws,
+          rpcService,
+          runId,
+          step.stepId,
+          'a',
+          'doA',
+          {},
+          'testQueuedOnErrorThrow'
+        ),
+      /boom/
+    )
+
+    const errorStep = await ws.getStepState(runId, 'onErr')
+    assert.notEqual(
+      errorStep.stepId,
+      '',
+      'onError node should be queued as a step'
+    )
+
+    delete metaState['testQueuedOnErrorThrow']
+  })
+
+  test('executeWorkflowStep should mark graph step failed and suspend run on RPCNotFoundError', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testQueuedRpcMissing'] = {
+      name: 'testQueuedRpcMissing',
+      pikkuFuncId: 'testQueuedRpcMissing',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'queued-rpc-missing-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'missingRpc' },
+      },
+    }
+
+    const runId = await ws.createRun(
+      'testQueuedRpcMissing',
+      {},
+      false,
+      'queued-rpc-missing-hash',
+      { type: 'test' }
+    )
+    await ws.insertStepState(runId, 'a', 'missingRpc', {})
+
+    const rpcService = {
+      rpcWithWire: async () => {
+        throw new RPCNotFoundError('missingRpc')
+      },
+    }
+
+    await ws.executeWorkflowStep(runId, 'a', 'missingRpc', {}, rpcService)
+
+    const step = await ws.getStepState(runId, 'a')
+    const run = await ws.getRun(runId)
+
+    assert.equal(step.status, 'failed')
+    assert.equal(step.error?.message, 'RPC function not found: missingRpc')
+    assert.equal(run?.status, 'suspended')
+    assert.equal(run?.error?.code, 'RPC_NOT_FOUND')
+
+    delete metaState['testQueuedRpcMissing']
+  })
+
+  test('inline graph missing RPC should mark step failed and suspend run', async () => {
+    const ws = new InMemoryWorkflowService()
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    metaState['testInlineRpcMissing'] = {
+      name: 'testInlineRpcMissing',
+      pikkuFuncId: 'testInlineRpcMissing',
+      source: 'graph',
+      entryNodeIds: ['a'],
+      graphHash: 'inline-rpc-missing-hash',
+      nodes: {
+        a: { nodeId: 'a', rpcName: 'missingInlineRpc' },
+      },
+    }
+
+    const rpcService = {
+      rpcWithWire: async () => {
+        throw new RPCNotFoundError('missingInlineRpc')
+      },
+    }
+
+    const { runId } = await runWorkflowGraph(
+      ws,
+      'testInlineRpcMissing',
+      {},
+      rpcService,
+      true
+    )
+
+    const step = await ws.getStepState(runId, 'a')
+    const run = await ws.getRun(runId)
+
+    assert.equal(step.status, 'failed')
+    assert.equal(
+      step.error?.message,
+      'RPC function not found: missingInlineRpc'
+    )
+    assert.equal(run?.status, 'suspended')
+    assert.equal(run?.error?.code, 'RPC_NOT_FOUND')
+
+    delete metaState['testInlineRpcMissing']
+  })
+})