ocpipe 0.1.0

package/pipeline.ts

@@ -0,0 +1,213 @@
+/**
+ * DSTS SDK Pipeline orchestrator.
+ *
+ * Manages execution context, state, checkpointing, logging, retry logic, and sub-pipelines.
+ */
+
+import { writeFile, readFile, readdir, mkdir } from 'fs/promises'
+import type { Module } from './module.js'
+import type {
+  BaseState,
+  ExecutionContext,
+  PipelineConfig,
+  RunOptions,
+  StepResult,
+} from './types.js'
+import { logStep } from './agent.js'
+import { JsonParseError, SchemaValidationError } from './parsing.js'
+
+/** Pipeline orchestrates workflow execution with state management. */
+export class Pipeline<S extends BaseState> {
+  state: S
+  private ctx: ExecutionContext
+  private stepNumber = 0
+
+  constructor(
+    public readonly config: PipelineConfig,
+    createState: () => S,
+  ) {
+    this.state = createState()
+    this.ctx = {
+      sessionId: undefined,
+      defaultModel: config.defaultModel,
+      defaultAgent: config.defaultAgent,
+      timeoutSec: config.timeoutSec ?? 300,
+    }
+  }
+
+  /** run executes a module and records the result. */
+  async run<I, O>(
+    module: Module<I, O>,
+    input: I,
+    options?: RunOptions,
+  ): Promise<StepResult<O>> {
+    const stepName = options?.name ?? module.constructor.name
+    this.stepNumber++
+    logStep(this.stepNumber, stepName)
+
+    // Handle session control
+    if (options?.newSession) {
+      this.ctx.sessionId = undefined
+    }
+
+    // Handle model override
+    const originalModel = this.ctx.defaultModel
+    if (options?.model) {
+      this.ctx.defaultModel = options.model
+    }
+
+    const startTime = Date.now()
+    let lastError: Error | undefined
+    const retryConfig = options?.retry ?? this.config.retry ?? { maxAttempts: 1 }
+
+    for (let attempt = 1; attempt <= retryConfig.maxAttempts; attempt++) {
+      try {
+        const data = await module.forward(input, this.ctx)
+
+        // Restore original model
+        this.ctx.defaultModel = originalModel
+
+        const result: StepResult<O> = {
+          data,
+          stepName,
+          duration: Date.now() - startTime,
+          sessionId: this.ctx.sessionId ?? '',
+          model: options?.model ?? this.config.defaultModel,
+          attempt,
+        }
+
+        // Record step for checkpointing
+        this.state.steps.push({
+          stepName,
+          timestamp: new Date().toISOString(),
+          result: result as StepResult<unknown>,
+        })
+
+        // Update opencode session in state
+        this.state.opencodeSessionId = this.ctx.sessionId
+
+        await this.saveCheckpoint()
+        return result
+      } catch (err) {
+        lastError = err as Error
+        const isParseError = err instanceof JsonParseError
+        const isSchemaError = err instanceof SchemaValidationError
+
+        // Don't retry SchemaValidationError - corrections already attempted
+        if (isSchemaError) {
+          console.error(
+            `Step ${stepName} failed with schema validation error (corrections exhausted): ${lastError.message}`,
+          )
+          break
+        }
+
+        if (
+          attempt < retryConfig.maxAttempts &&
+          (!isParseError || retryConfig.onParseError)
+        ) {
+          console.error(
+            `Step ${stepName} failed (attempt ${attempt}/${retryConfig.maxAttempts}): ${lastError.message}`,
+          )
+          continue
+        }
+        break
+      }
+    }
+
+    // Save checkpoint before throwing
+    await this.saveCheckpoint()
+    throw lastError
+  }
+
+  /** runSub executes a sub-pipeline with its own session. */
+  async runSub<SS extends BaseState, T>(
+    subConfig: PipelineConfig,
+    createState: () => SS,
+    executor: (sub: Pipeline<SS>) => Promise<T>,
+  ): Promise<StepResult<T>> {
+    this.stepNumber++
+    logStep(this.stepNumber, `sub:${subConfig.name}`)
+
+    const subPipeline = new Pipeline(subConfig, createState)
+    const startTime = Date.now()
+
+    const data = await executor(subPipeline)
+
+    // Record sub-pipeline reference
+    this.state.subPipelines.push({
+      name: subConfig.name,
+      sessionId: subPipeline.ctx.sessionId ?? '',
+      timestamp: new Date().toISOString(),
+      state: subPipeline.state,
+    })
+
+    await this.saveCheckpoint()
+
+    return {
+      data,
+      stepName: `sub:${subConfig.name}`,
+      duration: Date.now() - startTime,
+      sessionId: subPipeline.ctx.sessionId ?? '',
+      model: subConfig.defaultModel,
+      attempt: 1,
+    }
+  }
+
+  /** getSessionId returns the current OpenCode session ID. */
+  getSessionId(): string | undefined {
+    return this.ctx.sessionId
+  }
+
+  /** setPhase updates the current phase in state. */
+  setPhase(phase: string): void {
+    this.state.phase = phase
+  }
+
+  /** saveCheckpoint persists the current state to disk. */
+  async saveCheckpoint(): Promise<void> {
+    await mkdir(this.config.checkpointDir, { recursive: true })
+    const path = `${this.config.checkpointDir}/${this.config.name}_${this.state.sessionId}.json`
+    await writeFile(path, JSON.stringify(this.state, null, 2))
+  }
+
+  /** loadCheckpoint loads a pipeline from a checkpoint file. */
+  static async loadCheckpoint<S extends BaseState>(
+    config: PipelineConfig,
+    sessionId: string,
+  ): Promise<Pipeline<S> | null> {
+    const path = `${config.checkpointDir}/${config.name}_${sessionId}.json`
+    
+    try {
+      const content = await readFile(path, 'utf-8')
+      const state = JSON.parse(content) as S
+      const pipeline = new Pipeline<S>(config, () => state)
+      pipeline.state = state
+
+      // Restore context from state
+      if (state.opencodeSessionId) {
+        pipeline.ctx.sessionId = state.opencodeSessionId
+      }
+
+      // Restore step number
+      pipeline.stepNumber = state.steps.length
+
+      return pipeline
+    } catch {
+      return null
+    }
+  }
+
+  /** listCheckpoints lists all checkpoint files for a pipeline name. */
+  static async listCheckpoints(config: PipelineConfig): Promise<string[]> {
+    try {
+      const allFiles = await readdir(config.checkpointDir)
+      const prefix = `${config.name}_`
+      const files = allFiles
+        .filter((f) => f.startsWith(prefix) && f.endsWith('.json'))
+        .map((f) => `${config.checkpointDir}/${f}`)
+      return files.sort().reverse() // Most recent first
+    } catch {
+      return []
+    }
+  }
+}

package/predict.ts

@@ -0,0 +1,271 @@
+/**
+ * DSTS SDK Predict class.
+ *
+ * Executes a signature by generating a prompt, calling OpenCode, and parsing the response.
+ */
+
+import { z } from 'zod'
+import type {
+  CorrectionConfig,
+  CorrectionMethod,
+  ExecutionContext,
+  FieldConfig,
+  FieldError,
+  InferInputs,
+  InferOutputs,
+  ModelConfig,
+  PredictResult,
+  SignatureDef,
+} from './types.js'
+import { runAgent } from './agent.js'
+import {
+  tryParseResponse,
+  // jq-style patches
+  buildPatchPrompt,
+  buildBatchPatchPrompt,
+  extractPatch,
+  applyJqPatch,
+  // JSON Patch (RFC 6902)
+  buildJsonPatchPrompt,
+  buildBatchJsonPatchPrompt,
+  extractJsonPatch,
+  applyJsonPatch,
+  SchemaValidationError,
+} from './parsing.js'
+
+/** Configuration for a Predict instance. */
+export interface PredictConfig {
+  /** Override the pipeline's default agent. */
+  agent?: string
+  /** Override the pipeline's default model. */
+  model?: ModelConfig
+  /** Start a fresh session (default: false, reuses context). */
+  newSession?: boolean
+  /** Custom prompt template function. */
+  template?: (inputs: Record<string, unknown>) => string
+  /** Schema correction options (enabled by default, set to false to disable). */
+  correction?: CorrectionConfig | false
+}
+
+/** Predict executes a signature by calling an LLM and parsing the response. */
+export class Predict<S extends SignatureDef<any, any>> {
+  constructor(
+    public readonly sig: S,
+    public readonly config: PredictConfig = {},
+  ) {}
+
+  /** execute runs the prediction with the given inputs. */
+  async execute(
+    inputs: InferInputs<S>,
+    ctx: ExecutionContext,
+  ): Promise<PredictResult<InferOutputs<S>>> {
+    const prompt = this.buildPrompt(inputs as Record<string, unknown>)
+    const startTime = Date.now()
+
+    const agentResult = await runAgent({
+      prompt,
+      agent: this.config.agent ?? ctx.defaultAgent,
+      model: this.config.model ?? ctx.defaultModel,
+      sessionId: this.config.newSession ? undefined : ctx.sessionId,
+      timeoutSec: ctx.timeoutSec,
+    })
+
+    // Update context with new session ID for continuity
+    ctx.sessionId = agentResult.sessionId
+
+    const parseResult = tryParseResponse<InferOutputs<S>>(
+      agentResult.text,
+      this.sig.outputs,
+    )
+
+    // If parsing succeeded, return the result
+    if (parseResult.ok && parseResult.data) {
+      return {
+        data: parseResult.data,
+        raw: agentResult.text,
+        sessionId: agentResult.sessionId,
+        duration: Date.now() - startTime,
+        model: this.config.model ?? ctx.defaultModel,
+      }
+    }
+
+    // Parsing failed - attempt correction if enabled
+    if (this.config.correction !== false && parseResult.errors && parseResult.json) {
+      const corrected = await this.correctFields(
+        parseResult.json,
+        parseResult.errors,
+        ctx,
+        agentResult.sessionId,
+      )
+
+      if (corrected) {
+        return {
+          data: corrected,
+          raw: agentResult.text,
+          sessionId: agentResult.sessionId,
+          duration: Date.now() - startTime,
+          model: this.config.model ?? ctx.defaultModel,
+        }
+      }
+    }
+
+    // Correction failed or disabled - throw SchemaValidationError (non-retryable)
+    const errors = parseResult.errors ?? []
+    const errorMessages = errors.map((e) => `${e.path}: ${e.message}`).join('; ') || 'Unknown error'
+    const correctionAttempts = this.config.correction !== false ? (typeof this.config.correction === 'object' ? this.config.correction.maxRounds ?? 3 : 3) : 0
+    throw new SchemaValidationError(`Schema validation failed: ${errorMessages}`, errors, correctionAttempts)
+  }
+
+  /** correctFields attempts to fix field errors using same-session patches with retries. */
+  private async correctFields(
+    json: Record<string, unknown>,
+    initialErrors: FieldError[],
+    ctx: ExecutionContext,
+    sessionId: string,
+  ): Promise<InferOutputs<S> | null> {
+    const correctionConfig = typeof this.config.correction === 'object' ? this.config.correction : {}
+    const method: CorrectionMethod = correctionConfig.method ?? 'json-patch'
+    const maxFields = correctionConfig.maxFields ?? 5
+    const maxRounds = correctionConfig.maxRounds ?? 3
+    const correctionModel = correctionConfig.model
+
+    let currentJson = JSON.parse(JSON.stringify(json)) as Record<string, unknown>
+    let currentErrors = initialErrors
+
+    for (let round = 1; round <= maxRounds; round++) {
+      const errorsToFix = currentErrors.slice(0, maxFields)
+      
+      if (errorsToFix.length === 0) {
+        break
+      }
+
+      console.error(`\n>>> Correction round ${round}/${maxRounds} [${method}]: fixing ${errorsToFix.length} field(s)...`)
+
+      // Build prompt based on correction method
+      const patchPrompt = method === 'jq'
+        ? (errorsToFix.length === 1
+            ? buildPatchPrompt(errorsToFix[0]!, currentJson, this.sig.outputs)
+            : buildBatchPatchPrompt(errorsToFix, currentJson))
+        : (errorsToFix.length === 1
+            ? buildJsonPatchPrompt(errorsToFix[0]!, currentJson, this.sig.outputs)
+            : buildBatchJsonPatchPrompt(errorsToFix, currentJson))
+
+      // Use same session (model has context) unless correction model specified
+      const patchResult = await runAgent({
+        prompt: patchPrompt,
+        model: correctionModel ?? ctx.defaultModel,
+        sessionId: correctionModel ? undefined : sessionId,
+        agent: ctx.defaultAgent,
+        timeoutSec: 60, // Short timeout for simple patches
+      })
+
+      // Extract and apply the patch based on method
+      if (method === 'jq') {
+        const patch = extractPatch(patchResult.text)
+        console.error(`  jq patch: ${patch}`)
+        currentJson = applyJqPatch(currentJson, patch)
+      } else {
+        const operations = extractJsonPatch(patchResult.text)
+        console.error(`  JSON Patch: ${JSON.stringify(operations)}`)
+        currentJson = applyJsonPatch(currentJson, operations)
+      }
+
+      // Re-validate the corrected JSON
+      const revalidated = tryParseResponse<InferOutputs<S>>(
+        JSON.stringify(currentJson),
+        this.sig.outputs,
+      )
+
+      if (revalidated.ok && revalidated.data) {
+        console.error(`  Schema correction successful after ${round} round(s)!`)
+        return revalidated.data
+      }
+
+      // Update errors for next round
+      currentErrors = revalidated.errors ?? []
+      
+      if (currentErrors.length === 0) {
+        // No errors but also no data? Shouldn't happen, but handle gracefully
+        console.error(`  Unexpected state: no errors but validation failed`)
+        break
+      }
+
+      console.error(`  Round ${round} complete, ${currentErrors.length} error(s) remaining`)
+    }
+
+    console.error(`  Schema correction failed after ${maxRounds} rounds`)
+    return null
+  }
+
+  /** buildPrompt generates the prompt from inputs. */
+  private buildPrompt(inputs: Record<string, unknown>): string {
+    // Allow custom template override
+    if (this.config.template) {
+      return this.config.template(inputs)
+    }
+    return this.generatePrompt(inputs)
+  }
+
+  /** generatePrompt creates a structured prompt from the signature and inputs. */
+  private generatePrompt(inputs: Record<string, unknown>): string {
+    const lines: string[] = []
+
+    // Task description from signature doc
+    lines.push(this.sig.doc)
+    lines.push('')
+
+    // Input fields as JSON
+    const inputsWithDescriptions: Record<string, unknown> = {}
+    for (const [name, config] of Object.entries(this.sig.inputs) as [
+      string,
+      FieldConfig,
+    ][]) {
+      inputsWithDescriptions[name] = inputs[name]
+    }
+    lines.push('INPUTS:')
+    lines.push('```json')
+    lines.push(JSON.stringify(inputsWithDescriptions, null, 2))
+    lines.push('```')
+    lines.push('')
+
+    // Output format with JSON Schema
+    lines.push('OUTPUT FORMAT:')
+    lines.push('Return a JSON object matching this schema EXACTLY.')
+    lines.push('IMPORTANT: For optional fields, OMIT the field entirely - do NOT use null.')
+    lines.push('')
+    lines.push('```json')
+    lines.push(JSON.stringify(this.buildOutputJsonSchema(), null, 2))
+    lines.push('```')
+
+    return lines.join('\n')
+  }
+
+  /** buildOutputJsonSchema creates a JSON Schema from the output field definitions. */
+  private buildOutputJsonSchema(): Record<string, unknown> {
+    // Build a Zod object from the output fields
+    const shape: Record<string, z.ZodType> = {}
+    for (const [name, config] of Object.entries(this.sig.outputs) as [string, FieldConfig][]) {
+      shape[name] = config.type
+    }
+    const outputSchema = z.object(shape)
+
+    // Convert to JSON Schema
+    const jsonSchema = z.toJSONSchema(outputSchema)
+
+    // Add field descriptions from our config (toJSONSchema uses .describe() metadata)
+    // Since our FieldConfig has a separate desc field, merge it in
+    const props = jsonSchema.properties as Record<string, Record<string, unknown>> | undefined
+    if (props) {
+      for (const [name, config] of Object.entries(this.sig.outputs) as [string, FieldConfig][]) {
+        if (config.desc && props[name]) {
+          // Only add if not already set by .describe()
+          if (!props[name].description) {
+            props[name].description = config.desc
+          }
+        }
+      }
+    }
+
+    return jsonSchema
+  }
+}

package/signature.ts

@@ -0,0 +1,97 @@
+/**
+ * DSTS SDK signature definition.
+ *
+ * Signatures declare input/output contracts for LLM interactions using Zod for validation.
+ */
+
+import { z } from 'zod'
+import type { FieldConfig, SignatureDef } from './types.js'
+
+/** signature creates a new signature definition. */
+export function signature<
+  I extends Record<string, FieldConfig>,
+  O extends Record<string, FieldConfig>,
+>(def: { doc: string; inputs: I; outputs: O }): SignatureDef<I, O> {
+  return def
+}
+
+/** field provides helper functions for creating common field configurations. */
+export const field = {
+  /** string creates a string field. */
+  string: (desc?: string): FieldConfig<z.ZodString> => ({
+    type: z.string(),
+    desc,
+  }),
+
+  /** number creates a number field. */
+  number: (desc?: string): FieldConfig<z.ZodNumber> => ({
+    type: z.number(),
+    desc,
+  }),
+
+  /** boolean creates a boolean field. */
+  boolean: (desc?: string): FieldConfig<z.ZodBoolean> => ({
+    type: z.boolean(),
+    desc,
+  }),
+
+  /** array creates an array field with the specified item type. */
+  array: <T extends z.ZodType>(
+    itemType: T,
+    desc?: string,
+  ): FieldConfig<z.ZodArray<T>> => ({
+    type: z.array(itemType),
+    desc,
+  }),
+
+  /** object creates an object field with the specified shape. */
+  object: <T extends z.ZodRawShape>(
+    shape: T,
+    desc?: string,
+  ): FieldConfig<z.ZodObject<T>> => ({
+    type: z.object(shape),
+    desc,
+  }),
+
+  /** enum creates an enum field with the specified values. */
+  enum: <const T extends readonly [string, ...string[]]>(
+    values: T,
+    desc?: string,
+  ) => ({
+    type: z.enum(values),
+    desc,
+  }),
+
+  /** optional wraps a field type to make it optional. */
+  optional: <T extends z.ZodType>(
+    fieldConfig: FieldConfig<T>,
+  ): FieldConfig<z.ZodOptional<T>> => ({
+    type: fieldConfig.type.optional(),
+    desc: fieldConfig.desc,
+  }),
+
+  /** nullable wraps a field type to make it nullable. */
+  nullable: <T extends z.ZodType>(
+    fieldConfig: FieldConfig<T>,
+  ): FieldConfig<z.ZodNullable<T>> => ({
+    type: fieldConfig.type.nullable(),
+    desc: fieldConfig.desc,
+  }),
+
+  /** custom creates a field with a custom Zod type. */
+  custom: <T extends z.ZodType>(type: T, desc?: string): FieldConfig<T> => ({
+    type,
+    desc,
+  }),
+}
+
+/** buildOutputSchema creates a Zod object schema from output field definitions. */
+export function buildOutputSchema<O extends Record<string, FieldConfig>>(
+  outputs: O,
+): z.ZodObject<{ [K in keyof O]: O[K]['type'] }> {
+  const shape = {} as { [K in keyof O]: O[K]['type'] }
+  for (const [key, config] of Object.entries(outputs)) {
+    ;(shape as Record<string, z.ZodType>)[key] = config.type
+  }
+  return z.object(shape)
+}

package/state.ts

@@ -0,0 +1,39 @@
+/**
+ * DSTS SDK state management.
+ *
+ * Provides base state types and helpers for checkpointable workflow state.
+ */
+
+import type { BaseState } from './types.js'
+
+/** createSessionId generates a unique session ID based on the current timestamp. */
+export function createSessionId(): string {
+  const now = new Date()
+  const id = now
+    .toISOString()
+    .replace(/[-:]/g, '')
+    .replace('T', '_')
+    .split('.')[0]
+  return id ?? ''
+}
+
+/** createBaseState creates a new base state with default values. */
+export function createBaseState(): BaseState {
+  return {
+    sessionId: createSessionId(),
+    startedAt: new Date().toISOString(),
+    phase: 'init',
+    steps: [],
+    subPipelines: [],
+  }
+}
+
+/** extendBaseState creates a state factory that extends BaseState with additional fields. */
+export function extendBaseState<T extends Record<string, unknown>>(
+  extension: T,
+): BaseState & T {
+  return {
+    ...createBaseState(),
+    ...extension,
+  }
+}