@pikku/core 0.11.1 → 0.11.2

package/src/services/workflow-service.ts

@@ -0,0 +1,74 @@
+import {
+  SerializedError,
+  CoreSingletonServices,
+  CreateWireServices,
+  CoreConfig,
+} from '../types/core.types.js'
+import {
+  WorkflowRun,
+  StepState,
+  WorkflowStatus,
+} from '../wirings/workflow/workflow.types.js'
+
+/**
+ * Interface for workflow orchestration
+ * Handles workflow execution, replay, orchestration logic, and run-level state
+ */
+export interface WorkflowService {
+  // Run-level state operations
+  createRun(workflowName: string, input: any, inline?: boolean): Promise<string>
+  getRun(id: string): Promise<WorkflowRun | null>
+  getRunHistory(runId: string): Promise<Array<StepState & { stepName: string }>>
+  updateRunStatus(
+    id: string,
+    status: WorkflowStatus,
+    output?: any,
+    error?: SerializedError
+  ): Promise<void>
+  withRunLock<T>(id: string, fn: () => Promise<T>): Promise<T>
+  close(): Promise<void>
+
+  // Orchestration operations
+  resumeWorkflow(runId: string): Promise<void>
+  setServices(
+    singletonServices: CoreSingletonServices,
+    createWireServices: CreateWireServices,
+    config: CoreConfig
+  ): void
+  startWorkflow<I>(
+    name: string,
+    input: I,
+    rpcService: any,
+    options?: { inline?: boolean; startNode?: string }
+  ): Promise<{ runId: string }>
+  runWorkflowJob(runId: string, rpcService: any): Promise<void>
+  orchestrateWorkflow(runId: string, rpcService: any): Promise<void>
+  executeWorkflowSleepCompleted(runId: string, stepId: string): Promise<void>
+
+  // Step-level state operations
+  insertStepState(
+    runId: string,
+    stepName: string,
+    rpcName: string,
+    data: any,
+    stepOptions?: { retries?: number; retryDelay?: string | number }
+  ): Promise<StepState>
+  getStepState(runId: string, stepName: string): Promise<StepState>
+  setStepRunning(stepId: string): Promise<void>
+  setStepScheduled(stepId: string): Promise<void>
+  setStepResult(stepId: string, result: any): Promise<void>
+  setStepError(stepId: string, error: Error): Promise<void>
+  createRetryAttempt(
+    stepId: string,
+    status: 'pending' | 'running'
+  ): Promise<StepState>
+
+  // Step execution
+  executeWorkflowStep(
+    runId: string,
+    stepName: string,
+    rpcName: string | null,
+    data: any,
+    rpcService: any
+  ): Promise<void>
+}

package/src/types/core.types.ts

@@ -1,5 +1,6 @@
 import type { Logger, LogLevel } from '../services/logger.js'
 import { VariablesService } from '../services/variables-service.js'
+import { SecretService } from '../services/secret-service.js'
 import { SchemaService } from '../services/schema-service.js'
 import { JWTService } from '../services/jwt-service.js'
 import { PikkuHTTP } from '../wirings/http/http.types.js'
@@ -16,6 +17,8 @@ import {
   WorkflowServiceConfig,
   WorkflowStepWire,
 } from '../wirings/workflow/workflow.types.js'
+import type { PikkuGraphWire } from '../wirings/workflow/graph/workflow-graph.types.js'
+import { PikkuTrigger } from '../wirings/trigger/trigger.types.js'
 import { SchedulerService } from '../services/scheduler-service.js'
 
 export type PikkuWiringTypes =
@@ -169,6 +172,8 @@ export interface CoreSingletonServices<Config extends CoreConfig = CoreConfig> {
   logger: Logger
   /** The variable service to be used */
   variables: VariablesService
+  /** The secrets service to retrieve secrets */
+  secrets: SecretService
   /** The workflow orchestrator service */
   workflowService?: WorkflowService
   /** The queue service */
@@ -189,6 +194,7 @@ export type PikkuWire<
   IsChannel extends true | null = null,
   MCPTools extends string | never = never,
   TypedWorkflow extends PikkuWorkflowWire | never = PikkuWorkflowWire,
+  TriggerOutput = unknown,
 > = Partial<{
   http: PikkuHTTP<In>
   mcp: PikkuMCP<MCPTools>
@@ -201,6 +207,8 @@ export type PikkuWire<
   cli: PikkuCLI
   workflow: TypedWorkflow
   workflowStep: WorkflowStepWire
+  graph: PikkuGraphWire
+  trigger: PikkuTrigger<TriggerOutput>
   initialSession: HasInitialSession extends true
     ? UserSession
     : UserSession | undefined
@@ -231,8 +239,8 @@ export type CorePikkuMiddlewareConfig<
 > = {
   /** The middleware function */
   func: CorePikkuMiddleware<SingletonServices, UserSession>
-  /** Optional human-readable name for the middleware */
-  name?: string
+  /** Optional human-readable title for the middleware */
+  title?: string
   /** Optional description of what the middleware does */
   description?: string
 }
@@ -275,7 +283,7 @@ export type CorePikkuMiddlewareGroup<
  *
  * // Configuration object syntax with metadata
  * export const logMiddleware = pikkuMiddleware({
- *   name: 'Request Logger',
+ *   title: 'Request Logger',
  *   description: 'Logs request information',
  *   func: async ({ logger }, next) => {
  *     logger.info('Request started')
@@ -367,6 +375,7 @@ export type CreateConfig<
 export type CommonWireMeta = {
   pikkuFuncName: string
 
+  title?: string
   tags?: string[]
   summary?: string
   description?: string

package/src/types/state.types.ts

@@ -0,0 +1,195 @@
+import { PikkuError, ErrorDetails } from '../errors/error-handler.js'
+import {
+  CorePikkuFunctionConfig,
+  CorePermissionGroup,
+  CorePikkuPermission,
+} from '../function/functions.types.js'
+import { CoreChannel, ChannelsMeta } from '../wirings/channel/channel.types.js'
+import { CLIMeta, CLIProgramState } from '../wirings/cli/cli.types.js'
+import {
+  HTTPMethod,
+  CoreHTTPFunctionWiring,
+  HTTPWiringsMeta,
+} from '../wirings/http/http.types.js'
+import {
+  CoreMCPResource,
+  MCPResourceMeta,
+  CoreMCPTool,
+  MCPToolMeta,
+  CoreMCPPrompt,
+  MCPPromptMeta,
+} from '../wirings/mcp/mcp.types.js'
+import {
+  CoreQueueWorker,
+  QueueWorkersMeta,
+} from '../wirings/queue/queue.types.js'
+import {
+  CoreScheduledTask,
+  ScheduledTasksMeta,
+} from '../wirings/scheduler/scheduler.types.js'
+import {
+  CoreWorkflow,
+  WorkflowsRuntimeMeta,
+  WorkflowWires,
+} from '../wirings/workflow/workflow.types.js'
+import type {
+  WorkflowGraphDefinition,
+  GraphNodeConfig,
+} from '../wirings/workflow/graph/workflow-graph.types.js'
+import { CoreTrigger, TriggerMeta } from '../wirings/trigger/trigger.types.js'
+import {
+  FunctionsMeta,
+  CorePikkuMiddleware,
+  CorePikkuMiddlewareGroup,
+  FunctionServicesMeta,
+  CreateConfig,
+  CreateSingletonServices,
+  CreateWireServices,
+  CoreConfig,
+  CoreSingletonServices,
+  CoreServices,
+  CoreUserSession,
+} from './core.types.js'
+
+/**
+ * State structure for an individual package
+ */
+export interface PikkuPackageState {
+  function: {
+    meta: FunctionsMeta
+    functions: Map<string, CorePikkuFunctionConfig<any, any>>
+  }
+  rpc: {
+    meta: Record<string, string>
+    files: Map<
+      string,
+      {
+        exportedName: string
+        path: string
+      }
+    >
+    /** Maps namespace aliases to package names (e.g., 'ext' -> '@pikku/templates-function-external') */
+    externalPackages: Map<string, string>
+  }
+  http: {
+    middleware: Map<string, CorePikkuMiddleware<any, any>[]>
+    permissions: Map<string, CorePermissionGroup | CorePikkuPermission[]>
+    routes: Map<HTTPMethod, Map<string, CoreHTTPFunctionWiring<any, any, any>>>
+    meta: HTTPWiringsMeta
+  }
+  channel: {
+    channels: Map<string, CoreChannel<any, any>>
+    meta: ChannelsMeta
+  }
+  scheduler: {
+    tasks: Map<string, CoreScheduledTask>
+    meta: ScheduledTasksMeta
+  }
+  queue: {
+    registrations: Map<string, CoreQueueWorker>
+    meta: QueueWorkersMeta
+  }
+  workflows: {
+    registrations: Map<string, CoreWorkflow>
+    graphRegistrations: Map<string, WorkflowGraphDefinition<any>>
+    /** DSL workflow wirings (from wireWorkflow({ func: ... })) */
+    wirings: Map<any, { wires: WorkflowWires; func: any }>
+    /** Graph workflow wirings (from wireWorkflow({ graph: ... })) */
+    graphWirings: Map<
+      any,
+      { wires: WorkflowWires; graph: Record<string, GraphNodeConfig<string>> }
+    >
+    meta: WorkflowsRuntimeMeta
+  }
+  trigger: {
+    triggers: Map<string, CoreTrigger>
+    meta: TriggerMeta
+  }
+  mcp: {
+    resources: Map<string, CoreMCPResource>
+    resourcesMeta: MCPResourceMeta
+    tools: Map<string, CoreMCPTool>
+    toolsMeta: MCPToolMeta
+    prompts: Map<string, CoreMCPPrompt>
+    promptsMeta: MCPPromptMeta
+  }
+  cli: {
+    meta: CLIMeta | Record<string, any> // Backward compatible with old published CLI format
+    programs: Record<string, CLIProgramState>
+  }
+  middleware: {
+    tagGroup: Record<string, CorePikkuMiddlewareGroup>
+    httpGroup: Record<string, CorePikkuMiddlewareGroup>
+    tagGroupMeta: Record<
+      string,
+      {
+        exportName: string | null
+        sourceFile: string
+        position: number
+        services: FunctionServicesMeta
+        middlewareCount: number
+        isFactory: boolean
+      }
+    >
+    httpGroupMeta: Record<
+      string,
+      {
+        exportName: string | null
+        sourceFile: string
+        position: number
+        services: FunctionServicesMeta
+        middlewareCount: number
+        isFactory: boolean
+      }
+    >
+  }
+  permissions: {
+    tagGroup: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+    httpGroup: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+    tagGroupMeta: Record<
+      string,
+      {
+        exportName: string | null
+        sourceFile: string
+        position: number
+        services: FunctionServicesMeta
+        permissionCount: number
+        isFactory: boolean
+      }
+    >
+    httpGroupMeta: Record<
+      string,
+      {
+        exportName: string | null
+        sourceFile: string
+        position: number
+        services: FunctionServicesMeta
+        permissionCount: number
+        isFactory: boolean
+      }
+    >
+  }
+  misc: {
+    errors: Map<PikkuError, ErrorDetails>
+    schemas: Map<string, any>
+    middleware: Record<string, CorePikkuMiddleware[]>
+    permissions: Record<string, CorePermissionGroup | CorePikkuPermission[]>
+  }
+  package: {
+    /** Service factory functions for external packages */
+    factories: {
+      createConfig?: CreateConfig<CoreConfig>
+      createSingletonServices?: CreateSingletonServices<
+        CoreConfig,
+        CoreSingletonServices
+      >
+      createWireServices?: CreateWireServices<
+        CoreSingletonServices,
+        CoreServices,
+        CoreUserSession
+      >
+    } | null
+    /** Cached singleton services for this package */
+    singletonServices: CoreSingletonServices | null
+  }
+}

package/src/utils.ts

@@ -1,4 +1,5 @@
 import { Logger } from './services/logger.js'
+import { CoreSingletonServices } from './types/core.types.js'
 
 // TODO: WireServices probably needs it's own type
 // but is an issue for the future and will be tackled
@@ -57,3 +58,57 @@ export const freezeDedupe = <T>(
   }
   return Object.freeze(out)
 }
+
+/**
+ * Stop a single service by calling its stop method if it exists
+ */
+const stopService = async (
+  logger: Logger,
+  name: string,
+  service: any
+): Promise<void> => {
+  const stop = service?.stop
+  if (stop) {
+    logger.info(`Stopping singleton service: ${name}`)
+    try {
+      await stop.call(service)
+    } catch (e: any) {
+      logger.error(`Error stopping service ${name}:`, e)
+    }
+  }
+}
+
+/**
+ * Stop all singleton services, including external package services.
+ * External package services are stopped first, then the parent services.
+ *
+ * @param singletonServices - The parent singleton services to stop
+ */
+export const stopSingletonServices = async (
+  singletonServices: CoreSingletonServices
+): Promise<void> => {
+  const logger = singletonServices.logger
+
+  // First, stop all external package singleton services
+  if (globalThis.pikkuState) {
+    for (const [packageName, packageState] of globalThis.pikkuState) {
+      // Skip main package - we handle it separately
+      if (packageName === '__main__') continue
+
+      const packageServices = packageState.package?.singletonServices
+      if (packageServices) {
+        logger.info(`Stopping singleton services for package: ${packageName}`)
+        for (const [name, service] of Object.entries(packageServices)) {
+          await stopService(logger, `${packageName}/${name}`, service)
+        }
+        // Clear the cached services
+        packageState.package.singletonServices = null
+      }
+    }
+  }
+
+  // Then stop the parent singleton services
+  for (const [name, service] of Object.entries(singletonServices)) {
+    await stopService(logger, name, service)
+  }
+}

package/src/wirings/channel/channel-handler.ts

@@ -17,7 +17,7 @@ const getRouteMeta = (
   routingProperty?: string,
   routerValue?: string
 ): ChannelMessageMeta => {
-  const channelMeta = pikkuState('channel', 'meta')[channelName]
+  const channelMeta = pikkuState(null, 'channel', 'meta')[channelName]
   if (!channelMeta) {
     throw new Error(`Channel ${channelName} not found`)
   }

package/src/wirings/channel/channel-runner.ts

@@ -33,13 +33,13 @@ export const wireChannel = <
   >
 ) => {
   // Get the channel metadata
-  const channelsMeta = pikkuState('channel', 'meta')
+  const channelsMeta = pikkuState(null, 'channel', 'meta')
   const channelMeta = channelsMeta[channel.name]
   if (!channelMeta) {
     throw new Error(`Channel metadata not found for channel: ${channel.name}`)
   }
 
-  pikkuState('channel', 'channels').set(channel.name, channel as any)
+  pikkuState(null, 'channel', 'channels').set(channel.name, channel as any)
 
   // Register onConnect function if provided
   if (channel.onConnect && channelMeta.connect) {
@@ -89,7 +89,7 @@ export const wireChannel = <
   }
 
   // Store the channel configuration
-  pikkuState('channel', 'channels').set(channel.name, channel as any)
+  pikkuState(null, 'channel', 'channels').set(channel.name, channel as any)
 }
 
 const getMatchingChannelConfig = (path: string) => {
@@ -98,7 +98,7 @@ const getMatchingChannelConfig = (path: string) => {
     return null
   }
 
-  const meta = pikkuState('channel', 'meta')
+  const meta = pikkuState(null, 'channel', 'meta')
   const channelMeta = Object.values(meta).find(
     (channelConfig) => channelConfig.route === matchedPath.route
   )
@@ -106,7 +106,7 @@ const getMatchingChannelConfig = (path: string) => {
     return null
   }
 
-  const channels = pikkuState('channel', 'channels')
+  const channels = pikkuState(null, 'channel', 'channels')
   const channelConfig = channels.get(channelMeta.name)
   if (!channelConfig) {
     return null

package/src/wirings/channel/local/local-channel-runner.test.ts

@@ -101,10 +101,12 @@ const mockCreateWireServices = async () => ({
 
 beforeEach(() => {
   resetPikkuState()
+  httpRouter.reset()
 })
 
 afterEach(() => {
   resetPikkuState()
+  httpRouter.reset()
 })
 
 test('runChannel should return undefined and 404 if no matching channel is found', async () => {
@@ -130,7 +132,7 @@ test('runChannel should return undefined and 404 if no matching channel is found
 test('runChannel should return a channel handler if channel matches and no auth required', async () => {
   resetPikkuState()
 
-  pikkuState('channel', 'meta', {
+  pikkuState(null, 'channel', 'meta', {
     test: {
       name: 'test',
       route: '/test-channel',

package/src/wirings/channel/log-channels.ts

@@ -6,7 +6,7 @@ import { Logger } from '../../services/index.js'
  * @param logger - A logger for logging information.
  */
 export const logChannels = (logger: Logger) => {
-  const channels = pikkuState('channel', 'channels')
+  const channels = pikkuState(null, 'channel', 'channels')
   if (channels.size === 0) {
     logger.info('No channels added')
     return

package/src/wirings/channel/serverless/serverless-channel-runner.ts

@@ -35,9 +35,9 @@ const getVariablesForChannel = ({
   channelHandlerFactory: PikkuChannelHandlerFactory
   openingData?: unknown
 }) => {
-  const channels = pikkuState('channel', 'channels')
+  const channels = pikkuState(null, 'channel', 'channels')
   const channelConfig = channels.get(channelName)
-  const channelsMeta = pikkuState('channel', 'meta')
+  const channelsMeta = pikkuState(null, 'channel', 'meta')
   const meta = channelsMeta[channelName]
   if (!channelConfig) {
     throw new Error(`Channel not found: ${channelName}`)

package/src/wirings/cli/channel/cli-channel-runner.ts

@@ -1,5 +1,5 @@
 import { pikkuState } from '../../../pikku-state.js'
-import { CorePikkuCLIRender, CLIMeta } from '../cli.types.js'
+import { CorePikkuCLIRender, type CLIMeta } from '../cli.types.js'
 import { generateCommandHelp, parseCLIArguments } from '../command-parser.js'
 
 /**
@@ -27,12 +27,13 @@ export async function executeCLIViaChannel({
   defaultRenderer?: CorePikkuCLIRender<any>
 }): Promise<void> {
   // Get CLI metadata from state
-  const allCLIMeta = pikkuState('cli', 'meta') as unknown as CLIMeta | undefined
-  if (!allCLIMeta) {
+  const cliMeta = pikkuState(null, 'cli', 'meta')
+  if (!cliMeta || !('programs' in cliMeta)) {
     throw new Error(
       '[PKU342] CLI metadata not found. No CLI wirings were registered. See https://pikku.dev/docs/pikku-cli/errors/pku342 for more information.'
     )
   }
+  const allCLIMeta = cliMeta as CLIMeta
   const programMeta = allCLIMeta.programs[programName]
 
   if (!programMeta) {

package/src/wirings/cli/cli-runner.test.ts

@@ -30,7 +30,7 @@ describe('CLI Runner', () => {
 
   describe('runCLICommand', () => {
     test('should throw NotFoundError when program not found', async () => {
-      pikkuState('cli', 'meta', { programs: {}, renderers: {} })
+      pikkuState(null, 'cli', 'meta', { programs: {}, renderers: {} })
 
       await assert.rejects(
         async () =>
@@ -45,7 +45,7 @@ describe('CLI Runner', () => {
     })
 
     test('should throw NotFoundError when command not found', async () => {
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'test-cli': {
             program: 'test-cli',
@@ -56,7 +56,7 @@ describe('CLI Runner', () => {
         renderers: {},
       })
 
-      pikkuState('cli', 'programs', {
+      pikkuState(null, 'cli', 'programs', {
         'test-cli': {
           defaultRenderer: undefined,
           middleware: [],
@@ -84,7 +84,7 @@ describe('CLI Runner', () => {
       }
 
       // Setup metadata
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'test-cli': {
             program: 'test-cli',
@@ -102,7 +102,7 @@ describe('CLI Runner', () => {
         renderers: {},
       })
 
-      pikkuState('cli', 'programs', {
+      pikkuState(null, 'cli', 'programs', {
         'test-cli': {
           defaultRenderer: undefined,
           middleware: [],
@@ -110,7 +110,7 @@ describe('CLI Runner', () => {
         },
       })
 
-      pikkuState('function', 'meta', {
+      pikkuState(null, 'function', 'meta', {
         greetFunc: {
           pikkuFuncName: 'greetFunc',
           inputSchemaName: null,
@@ -148,7 +148,7 @@ describe('CLI Runner', () => {
         return { success: true }
       }
 
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'test-cli': {
             program: 'test-cli',
@@ -166,7 +166,7 @@ describe('CLI Runner', () => {
         renderers: {},
       })
 
-      pikkuState('cli', 'programs', {
+      pikkuState(null, 'cli', 'programs', {
         'test-cli': {
           defaultRenderer: undefined,
           middleware: [testMiddleware],
@@ -174,7 +174,7 @@ describe('CLI Runner', () => {
         },
       })
 
-      pikkuState('function', 'meta', {
+      pikkuState(null, 'function', 'meta', {
         testFunc: {
           pikkuFuncName: 'testFunc',
           inputSchemaName: null,
@@ -208,7 +208,7 @@ describe('CLI Runner', () => {
 
       const testFunc = async () => ({ success: true })
 
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'test-cli': {
             program: 'test-cli',
@@ -226,7 +226,7 @@ describe('CLI Runner', () => {
         renderers: {},
       })
 
-      pikkuState('cli', 'programs', {
+      pikkuState(null, 'cli', 'programs', {
         'test-cli': {
           defaultRenderer: undefined,
           middleware: [testMiddleware],
@@ -234,7 +234,7 @@ describe('CLI Runner', () => {
         },
       })
 
-      pikkuState('function', 'meta', {
+      pikkuState(null, 'function', 'meta', {
         greetFunc: {
           pikkuFuncName: 'greetFunc',
           inputSchemaName: null,
@@ -260,7 +260,7 @@ describe('CLI Runner', () => {
     test('should throw error when auth required but no session', async () => {
       const testFunc = async () => ({ success: true })
 
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'test-cli': {
             program: 'test-cli',
@@ -278,7 +278,7 @@ describe('CLI Runner', () => {
         renderers: {},
       })
 
-      pikkuState('function', 'meta', {
+      pikkuState(null, 'function', 'meta', {
         secureFunc: {
           pikkuFuncName: 'secureFunc',
           inputSchemaName: null,
@@ -306,7 +306,7 @@ describe('CLI Runner', () => {
     test('should register CLI program and commands', () => {
       const greetFunc = async () => 'Hello'
 
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'my-cli': {
             program: 'my-cli',
@@ -337,7 +337,7 @@ describe('CLI Runner', () => {
         },
       })
 
-      const programs = pikkuState('cli', 'programs')
+      const programs = pikkuState(null, 'cli', 'programs')
       assert.ok(programs['my-cli'])
       assert.strictEqual(programs['my-cli'].middleware.length, 0)
     })
@@ -347,7 +347,7 @@ describe('CLI Runner', () => {
         await next()
       }
 
-      pikkuState('cli', 'meta', {
+      pikkuState(null, 'cli', 'meta', {
         programs: {
           'my-cli': {
             program: 'my-cli',
@@ -364,13 +364,13 @@ describe('CLI Runner', () => {
         middleware: [middleware],
       })
 
-      const programs = pikkuState('cli', 'programs')
+      const programs = pikkuState(null, 'cli', 'programs')
       assert.strictEqual(programs['my-cli'].middleware.length, 1)
       assert.strictEqual(programs['my-cli'].middleware[0], middleware)
     })
 
     test('should throw error when CLI metadata not found', () => {
-      pikkuState('cli', 'meta', { programs: {}, renderers: {} })
+      pikkuState(null, 'cli', 'meta', { programs: {}, renderers: {} })
 
       assert.throws(() => {
         wireCLI({