@pikku/core 0.11.2 → 0.12.1

package/src/services/in-memory-workflow-service.ts

@@ -0,0 +1,395 @@
+import { randomUUID } from 'crypto'
+import { PikkuWorkflowService } from '../wirings/workflow/pikku-workflow-service.js'
+import type { SerializedError } from '../types/core.types.js'
+import type {
+  WorkflowRun,
+  WorkflowRunWire,
+  StepState,
+  WorkflowStatus,
+  WorkflowStepOptions,
+} from '../wirings/workflow/workflow.types.js'
+
+interface InternalStepData {
+  rpcName: string | null
+  data: any
+  stepName: string
+}
+
+/**
+ * In-memory implementation of WorkflowService for inline-only execution
+ *
+ * This is a lightweight workflow service that stores all state in memory.
+ * It only supports inline execution (no queue workers) and is ideal for:
+ * - CLI tools that need workflow-like step orchestration
+ * - Testing and development
+ * - Single-process applications without persistence requirements
+ *
+ * @example
+ * ```typescript
+ * const workflowService = new InMemoryWorkflowService()
+ * await workflowService.startWorkflow('myWorkflow', input, { type: 'cli' }, rpc, { inline: true })
+ * ```
+ */
+export class InMemoryWorkflowService extends PikkuWorkflowService {
+  private runs = new Map<string, WorkflowRun>()
+  private steps = new Map<string, StepState>() // keyed by `${runId}:${stepName}`
+  private stepData = new Map<string, InternalStepData>() // keyed by stepId
+  private stepHistory = new Map<
+    string,
+    Array<StepState & { stepName: string }>
+  >() // keyed by runId
+  private runState = new Map<string, Record<string, unknown>>() // keyed by runId
+  private branchKeys = new Map<string, string>() // keyed by stepId
+  private workflowVersions = new Map<string, { graph: any; source: string }>() // keyed by `${name}:${graphHash}`
+
+  async createRun(
+    workflowName: string,
+    input: any,
+    inline: boolean,
+    graphHash: string,
+    wire: WorkflowRunWire
+  ): Promise<string> {
+    const runId = randomUUID()
+    const now = new Date()
+
+    const run: WorkflowRun = {
+      id: runId,
+      workflow: workflowName,
+      status: 'running',
+      input,
+      inline,
+      graphHash,
+      wire,
+      createdAt: now,
+      updatedAt: now,
+    }
+
+    this.runs.set(runId, run)
+    this.stepHistory.set(runId, [])
+    this.runState.set(runId, {})
+
+    if (inline) {
+      this.registerInlineRun(runId)
+    }
+
+    return runId
+  }
+
+  async getRun(id: string): Promise<WorkflowRun | null> {
+    return this.runs.get(id) || null
+  }
+
+  async getRunHistory(
+    runId: string
+  ): Promise<Array<StepState & { stepName: string }>> {
+    return this.stepHistory.get(runId) || []
+  }
+
+  async updateRunStatus(
+    id: string,
+    status: WorkflowStatus,
+    output?: any,
+    error?: SerializedError
+  ): Promise<void> {
+    const run = this.runs.get(id)
+    if (run) {
+      run.status = status
+      run.updatedAt = new Date()
+      if (output !== undefined) run.output = output
+      if (error !== undefined) run.error = error
+    }
+  }
+
+  async insertStepState(
+    runId: string,
+    stepName: string,
+    rpcName: string | null,
+    data: any,
+    stepOptions?: WorkflowStepOptions
+  ): Promise<StepState> {
+    const stepId = randomUUID()
+    const now = new Date()
+
+    const step: StepState = {
+      stepId,
+      status: 'pending',
+      attemptCount: 1,
+      retries: stepOptions?.retries,
+      retryDelay: stepOptions?.retryDelay,
+      createdAt: now,
+      updatedAt: now,
+    }
+
+    const key = `${runId}:${stepName}`
+    this.steps.set(key, step)
+    this.stepData.set(stepId, { rpcName, data, stepName })
+
+    // Add to history
+    const history = this.stepHistory.get(runId) || []
+    history.push({ ...step, stepName })
+    this.stepHistory.set(runId, history)
+
+    return step
+  }
+
+  async getStepState(runId: string, stepName: string): Promise<StepState> {
+    const key = `${runId}:${stepName}`
+    const step = this.steps.get(key)
+    if (!step) {
+      return {
+        stepId: '',
+        status: 'pending',
+        attemptCount: 0,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      }
+    }
+    return step
+  }
+
+  async setStepRunning(stepId: string): Promise<void> {
+    for (const step of this.steps.values()) {
+      if (step.stepId === stepId) {
+        step.status = 'running'
+        step.runningAt = new Date()
+        step.updatedAt = new Date()
+        break
+      }
+    }
+  }
+
+  async setStepScheduled(stepId: string): Promise<void> {
+    for (const step of this.steps.values()) {
+      if (step.stepId === stepId) {
+        step.status = 'scheduled'
+        step.scheduledAt = new Date()
+        step.updatedAt = new Date()
+        break
+      }
+    }
+  }
+
+  async setStepResult(stepId: string, result: any): Promise<void> {
+    for (const step of this.steps.values()) {
+      if (step.stepId === stepId) {
+        step.status = 'succeeded'
+        step.result = result
+        step.succeededAt = new Date()
+        step.updatedAt = new Date()
+        break
+      }
+    }
+  }
+
+  async setStepError(stepId: string, error: Error): Promise<void> {
+    for (const step of this.steps.values()) {
+      if (step.stepId === stepId) {
+        step.status = 'failed'
+        step.error = {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        }
+        step.failedAt = new Date()
+        step.updatedAt = new Date()
+        break
+      }
+    }
+  }
+
+  async createRetryAttempt(
+    failedStepId: string,
+    status: 'pending' | 'running'
+  ): Promise<StepState> {
+    // Find the failed step
+    let failedStep: StepState | undefined
+    let runId: string | undefined
+    let stepName: string | undefined
+
+    for (const [key, step] of this.steps.entries()) {
+      if (step.stepId === failedStepId) {
+        failedStep = step
+        const parts = key.split(':')
+        runId = parts[0]
+        stepName = parts.slice(1).join(':')
+        break
+      }
+    }
+
+    if (!failedStep || !runId || !stepName) {
+      throw new Error(`Step not found: ${failedStepId}`)
+    }
+
+    const failedStepData = this.stepData.get(failedStepId)
+    const newStepId = randomUUID()
+    const now = new Date()
+
+    const newStep: StepState = {
+      stepId: newStepId,
+      status,
+      attemptCount: failedStep.attemptCount + 1,
+      retries: failedStep.retries,
+      retryDelay: failedStep.retryDelay,
+      createdAt: now,
+      updatedAt: now,
+    }
+
+    if (status === 'running') {
+      newStep.runningAt = now
+    }
+
+    const key = `${runId}:${stepName}`
+    this.steps.set(key, newStep)
+
+    // Copy step data to new step
+    if (failedStepData) {
+      this.stepData.set(newStepId, { ...failedStepData })
+    }
+
+    // Add to history
+    const history = this.stepHistory.get(runId) || []
+    history.push({ ...newStep, stepName })
+    this.stepHistory.set(runId, history)
+
+    return newStep
+  }
+
+  async withRunLock<T>(_id: string, fn: () => Promise<T>): Promise<T> {
+    // In-memory service doesn't need locking for inline execution
+    return fn()
+  }
+
+  async withStepLock<T>(
+    _runId: string,
+    _stepName: string,
+    fn: () => Promise<T>
+  ): Promise<T> {
+    // In-memory service doesn't need locking for inline execution
+    return fn()
+  }
+
+  async close(): Promise<void> {
+    // Clear all in-memory state
+    this.runs.clear()
+    this.steps.clear()
+    this.stepData.clear()
+    this.stepHistory.clear()
+    this.runState.clear()
+    this.branchKeys.clear()
+    this.workflowVersions.clear()
+  }
+
+  // Graph workflow methods
+
+  async getCompletedGraphState(runId: string): Promise<{
+    completedNodeIds: string[]
+    failedNodeIds: string[]
+    branchKeys: Record<string, string>
+  }> {
+    const completedNodeIds: string[] = []
+    const failedNodeIds: string[] = []
+    const prefix = `${runId}:`
+    for (const [key, step] of this.steps.entries()) {
+      if (!key.startsWith(prefix)) continue
+      const nodeId = key.substring(prefix.length)
+      if (step.status === 'succeeded') {
+        completedNodeIds.push(nodeId)
+      } else if (step.status === 'failed') {
+        const maxAttempts = (step.retries ?? 0) + 1
+        if (step.attemptCount >= maxAttempts) {
+          failedNodeIds.push(nodeId)
+        }
+      }
+    }
+
+    const branchKeys: Record<string, string> = {}
+    for (const [stepId, branchKey] of this.branchKeys.entries()) {
+      const stepData = this.stepData.get(stepId)
+      if (stepData) {
+        branchKeys[stepData.stepName] = branchKey
+      }
+    }
+
+    return { completedNodeIds, failedNodeIds, branchKeys }
+  }
+
+  async getNodesWithoutSteps(
+    runId: string,
+    nodeIds: string[]
+  ): Promise<string[]> {
+    const existingSteps = new Set<string>()
+    const prefix = `${runId}:`
+    for (const [key] of this.steps.entries()) {
+      if (key.startsWith(prefix)) {
+        existingSteps.add(key.substring(prefix.length))
+      }
+    }
+    return nodeIds.filter((id) => !existingSteps.has(id))
+  }
+
+  async getNodeResults(
+    runId: string,
+    nodeIds: string[]
+  ): Promise<Record<string, any>> {
+    const results: Record<string, any> = {}
+    for (const nodeId of nodeIds) {
+      const key = `${runId}:${nodeId}`
+      const step = this.steps.get(key)
+      if (step?.result !== undefined) {
+        results[nodeId] = step.result
+      }
+    }
+    return results
+  }
+
+  async setBranchKey(
+    runId: string,
+    nodeId: string,
+    branchKey: string
+  ): Promise<void> {
+    const key = `${runId}:${nodeId}`
+    const step = this.steps.get(key)
+    if (step) {
+      this.branchKeys.set(step.stepId, branchKey)
+    }
+  }
+
+  async setBranchTaken(stepId: string, branchKey: string): Promise<void> {
+    this.branchKeys.set(stepId, branchKey)
+  }
+
+  async updateRunState(
+    runId: string,
+    name: string,
+    value: unknown
+  ): Promise<void> {
+    const state = this.runState.get(runId) || {}
+    state[name] = value
+    this.runState.set(runId, state)
+  }
+
+  async getRunState(runId: string): Promise<Record<string, unknown>> {
+    return this.runState.get(runId) || {}
+  }
+
+  async upsertWorkflowVersion(
+    name: string,
+    graphHash: string,
+    graph: any,
+    source: string
+  ): Promise<void> {
+    this.workflowVersions.set(`${name}:${graphHash}`, {
+      graph,
+      source,
+    })
+  }
+
+  async getWorkflowVersion(
+    name: string,
+    graphHash: string
+  ): Promise<{ graph: any; source: string } | null> {
+    const version = this.workflowVersions.get(`${name}:${graphHash}`)
+    if (!version) return null
+    return { graph: version.graph, source: version.source }
+  }
+}

package/src/services/index.ts

@@ -2,17 +2,53 @@
  * @module @pikku/core
  */
 
-// Interfaces
-export * from './content-service.js'
-export * from './jwt-service.js'
-export * from './logger.js'
-export * from './secret-service.js'
-export * from './variables-service.js'
-export * from './schema-service.js'
-export * from './user-session-service.js'
-export * from './scheduler-service.js'
-
-// Local implementations
-export * from './local-secrets.js'
-export * from './local-variables.js'
-export * from './logger-console.js'
+export { LogLevel } from './logger.js'
+export { ScopedSecretService } from './scoped-secret-service.js'
+export {
+  PikkuSessionService,
+  createMiddlewareSessionWireProps,
+  createFunctionSessionWireProps,
+} from './user-session-service.js'
+export { TypedSecretService } from './typed-secret-service.js'
+export { TypedVariablesService } from './typed-variables-service.js'
+export { LocalSecretService } from './local-secrets.js'
+export { LocalVariablesService } from './local-variables.js'
+export { ConsoleLogger } from './logger-console.js'
+export { InMemoryWorkflowService } from './in-memory-workflow-service.js'
+export { InMemoryTriggerService } from './in-memory-trigger-service.js'
+export { InMemoryAIRunStateService } from './in-memory-ai-run-state-service.js'
+export type { ContentService } from './content-service.js'
+export type { JWTService } from './jwt-service.js'
+export type { Logger } from './logger.js'
+export type { SecretService } from './secret-service.js'
+export type { VariablesService } from './variables-service.js'
+export type { SchemaService } from './schema-service.js'
+export type { SessionService } from './user-session-service.js'
+export type {
+  ScheduledTaskSummary,
+  ScheduledTaskInfo,
+  SchedulerService,
+} from './scheduler-service.js'
+export type { TriggerService } from './trigger-service.js'
+export type {
+  DeploymentService,
+  DeploymentConfig,
+  DeploymentInfo,
+  DeploymentServiceConfig,
+} from './deployment-service.js'
+export type { AIStorageService } from './ai-storage-service.js'
+export type {
+  AIAgentRunnerParams,
+  AIAgentRunnerResult,
+  AIAgentStepResult,
+  AIAgentRunnerService,
+} from './ai-agent-runner-service.js'
+export type {
+  CreateRunInput,
+  AIRunStateService,
+} from './ai-run-state-service.js'
+export type {
+  CredentialStatus,
+  CredentialMeta,
+} from './typed-secret-service.js'
+export type { VariableStatus, VariableMeta } from './typed-variables-service.js'

package/src/services/local-content.ts

@@ -1,8 +1,8 @@
 import { createReadStream, createWriteStream, promises } from 'fs'
-import { mkdir } from 'fs/promises'
-import { ContentService, Logger } from '@pikku/core/services'
+import { mkdir, readFile } from 'fs/promises'
+import type { ContentService, Logger } from '@pikku/core/services'
 import { pipeline } from 'stream/promises'
-import { Readable } from 'stream'
+import type { Readable } from 'stream'
 
 export interface LocalContentConfig {
   localFileUploadPath: string
@@ -96,6 +96,12 @@ export class LocalContent implements ContentService {
     }
   }
 
+  public async readFileAsBuffer(assetKey: string): Promise<Buffer> {
+    const filePath = `${this.config.localFileUploadPath}/${assetKey}`
+    this.logger.debug(`Reading file as buffer: ${assetKey}`)
+    return readFile(filePath)
+  }
+
   public async deleteFile(assetKey: string): Promise<boolean> {
     this.logger.debug(`deleting key: ${assetKey}`)
     try {

package/src/services/local-secrets.test.ts

@@ -0,0 +1,80 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert'
+import { LocalSecretService } from './local-secrets.js'
+import { LocalVariablesService } from './local-variables.js'
+
+describe('LocalSecretService', () => {
+  test('should get secret from local storage', async () => {
+    const service = new LocalSecretService()
+    await service.setSecretJSON('MY_KEY', { token: 'abc' })
+    const result = await service.getSecret('MY_KEY')
+    assert.strictEqual(result, '{"token":"abc"}')
+  })
+
+  test('should get secret from environment variables', async () => {
+    const vars = new LocalVariablesService({ MY_SECRET: 'secret-value' })
+    const service = new LocalSecretService(vars)
+    const result = await service.getSecret('MY_SECRET')
+    assert.strictEqual(result, 'secret-value')
+  })
+
+  test('should throw when secret not found', async () => {
+    const vars = new LocalVariablesService({})
+    const service = new LocalSecretService(vars)
+    await assert.rejects(() => service.getSecret('MISSING'), {
+      message: 'Secret Not Found: MISSING',
+    })
+  })
+
+  test('should get JSON secret from local storage', async () => {
+    const service = new LocalSecretService()
+    await service.setSecretJSON('JSON_KEY', { data: 42 })
+    const result = await service.getSecretJSON('JSON_KEY')
+    assert.deepStrictEqual(result, { data: 42 })
+  })
+
+  test('should get JSON secret from environment variables', async () => {
+    const vars = new LocalVariablesService({ CONFIG: '{"port":3000}' })
+    const service = new LocalSecretService(vars)
+    const result = await service.getSecretJSON('CONFIG')
+    assert.deepStrictEqual(result, { port: 3000 })
+  })
+
+  test('should throw when JSON secret not found', async () => {
+    const vars = new LocalVariablesService({})
+    const service = new LocalSecretService(vars)
+    await assert.rejects(() => service.getSecretJSON('MISSING'), {
+      message: 'Secret Not Found: MISSING',
+    })
+  })
+
+  test('should prefer local storage over env variables', async () => {
+    const vars = new LocalVariablesService({ KEY: 'env-value' })
+    const service = new LocalSecretService(vars)
+    await service.setSecretJSON('KEY', 'local-value')
+    const result = await service.getSecret('KEY')
+    assert.strictEqual(result, '"local-value"')
+  })
+
+  test('should check hasSecret in local storage', async () => {
+    const vars = new LocalVariablesService({})
+    const service = new LocalSecretService(vars)
+    await service.setSecretJSON('EXISTS', 'yes')
+    assert.strictEqual(await service.hasSecret('EXISTS'), true)
+    assert.strictEqual(await service.hasSecret('NOPE'), false)
+  })
+
+  test('should check hasSecret in environment', async () => {
+    const vars = new LocalVariablesService({ ENV_KEY: 'val' })
+    const service = new LocalSecretService(vars)
+    assert.strictEqual(await service.hasSecret('ENV_KEY'), true)
+  })
+
+  test('should delete secret from local storage', async () => {
+    const service = new LocalSecretService()
+    await service.setSecretJSON('DEL_KEY', 'value')
+    assert.strictEqual(await service.hasSecret('DEL_KEY'), true)
+    await service.deleteSecret('DEL_KEY')
+    assert.strictEqual(await service.hasSecret('DEL_KEY'), false)
+  })
+})

package/src/services/local-secrets.ts

@@ -1,11 +1,14 @@
 import { LocalVariablesService } from './local-variables.js'
-import { SecretService } from './secret-service.js'
-import { VariablesService } from './variables-service.js'
+import type { SecretService } from './secret-service.js'
+import type { VariablesService } from './variables-service.js'
 
 /**
  * Service for retrieving secrets from environment variables.
+ * Supports storing secrets locally in memory for CLI operations.
  */
 export class LocalSecretService implements SecretService {
+  private localSecrets: Map<string, string> = new Map()
+
   /**
    * Creates an instance of LocalSecretService.
    */
@@ -15,11 +18,19 @@ export class LocalSecretService implements SecretService {
 
   /**
    * Retrieves a secret by key.
+   * Checks local storage first, then falls back to environment variables.
    * @param key - The key of the secret to retrieve.
    * @returns A promise that resolves to the secret value.
    * @throws {Error} If the secret is not found.
    */
   public async getSecretJSON<R>(key: string): Promise<R> {
+    // Check local storage first
+    const localValue = this.localSecrets.get(key)
+    if (localValue) {
+      return JSON.parse(localValue)
+    }
+
+    // Fall back to environment variables
     const value = await this.variables.get(key)
     if (value) {
       return JSON.parse(value)
@@ -29,15 +40,55 @@ export class LocalSecretService implements SecretService {
 
   /**
    * Retrieves a secret by key.
+   * Checks local storage first, then falls back to environment variables.
    * @param key - The key of the secret to retrieve.
    * @returns A promise that resolves to the secret value.
    * @throws {Error} If the secret is not found.
    */
   public async getSecret(key: string): Promise<string> {
+    // Check local storage first
+    const localValue = this.localSecrets.get(key)
+    if (localValue) {
+      return localValue
+    }
+
+    // Fall back to environment variables
     const value = await this.variables.get(key)
     if (value) {
       return value
     }
     throw new Error(`Secret Not Found: ${key}`)
   }
+
+  /**
+   * Stores a JSON value as a secret in local storage.
+   * @param key - The key to store the secret under.
+   * @param value - The JSON value to store.
+   * @returns A promise that resolves when the secret is stored.
+   */
+  public async setSecretJSON(key: string, value: unknown): Promise<void> {
+    this.localSecrets.set(key, JSON.stringify(value))
+  }
+
+  /**
+   * Checks if a secret exists without throwing.
+   * @param key - The key of the secret to check.
+   * @returns A promise that resolves to true if the secret exists.
+   */
+  public async hasSecret(key: string): Promise<boolean> {
+    if (this.localSecrets.has(key)) {
+      return true
+    }
+    const value = await this.variables.get(key)
+    return value !== undefined && value !== null && value !== ''
+  }
+
+  /**
+   * Deletes a secret from local storage.
+   * @param key - The key of the secret to delete.
+   * @returns A promise that resolves when the secret is deleted.
+   */
+  public async deleteSecret(key: string): Promise<void> {
+    this.localSecrets.delete(key)
+  }
 }