ocpipe 0.1.0

package/README.md

@@ -0,0 +1,334 @@
+# DSTS
+
+<div align="center">
+  <h3>Declarative Self-Improving TypeScript</h3>
+  <p>A DSPy-inspired SDK for building LLM workflow pipelines with <a href="https://opencode.ai">OpenCode</a>.</p>
+  <p>
+    <a href="https://github.com/s4wave/dsts">GitHub</a> |
+    <a href="https://github.com/s4wave/dsts/blob/main/GETTING_STARTED.md">Getting Started</a> |
+    <a href="https://github.com/s4wave/dsts/blob/main/LICENSE">MIT License</a>
+  </p>
+</div>
+
+<div align="center">
+
+```
+Signature  →  Predict  →  Module  →  Pipeline
+   │            │           │           │
+ what        execute     compose    orchestrate
+```
+
+</div>
+
+DSTS separates the **what** (Signatures declare input/output contracts), the **how** (Modules compose predictors), and the **when** (Pipelines orchestrate execution). This separation enables clean composition, rich debugging, and maintainable LLM workflow code.
+
+## Features
+
+- **Type-safe signatures** - Define input/output contracts with Zod schemas
+- **Automatic prompt generation** - Signatures become structured prompts
+- **JSON output parsing** - Automatic extraction and validation of JSON responses
+- **Session continuity** - Reuse OpenCode sessions across steps
+- **Checkpointing** - Automatic state persistence after each step
+- **Retry logic** - Configurable retries with parse error handling
+- **Sub-pipelines** - Compose complex workflows from smaller pieces
+- **Testing utilities** - Mock backends for unit testing
+
+## Quick Start
+
+```typescript
+import { signature, field, SignatureModule, Pipeline, createBaseState } from 'dsts'
+import { z } from 'zod'
+
+// 1. Define a signature (the contract)
+const ParseIntent = signature({
+  doc: 'Parse user intent from a natural language description.',
+  inputs: {
+    description: field.string('User description in natural language'),
+  },
+  outputs: {
+    intent: field.string('Parsed intent category'),
+    confidence: field.number('Confidence score 0-1'),
+    keywords: field.array(z.string(), 'Extracted keywords'),
+  },
+})
+
+// 2. Create a module (the logic)
+class IntentParser extends SignatureModule<typeof ParseIntent> {
+  constructor() {
+    super(ParseIntent)
+  }
+
+  async forward(input, ctx) {
+    const result = await this.predictor.execute(input, ctx)
+    return result.data  // Full signature output
+  }
+}
+
+// 3. Run in a pipeline (the orchestration)
+const pipeline = new Pipeline({
+  name: 'my-workflow',
+  defaultModel: { providerID: 'anthropic', modelID: 'claude-sonnet-4-5' },
+  defaultAgent: 'general',
+  checkpointDir: './ckpt',
+  logDir: './logs',
+}, createBaseState)
+
+const result = await pipeline.run(new IntentParser(), { description: 'Hello world' })
+console.log(result.data.intent)
+```
+
+## Core Concepts
+
+### Signatures
+
+A Signature declares **what** an LLM interaction does - its inputs, outputs, and purpose. This is separate from *how* it executes.
+
+```typescript
+import { signature, field } from 'dsts'
+import { z } from 'zod'
+
+const AnalyzeCode = signature({
+  doc: 'Analyze code for potential issues and improvements.',
+  inputs: {
+    code: field.string('Source code to analyze'),
+    language: field.enum(['typescript', 'python', 'rust'] as const),
+  },
+  outputs: {
+    issues: field.array(z.object({
+      severity: z.enum(['error', 'warning', 'info']),
+      message: z.string(),
+      line: z.number(),
+    }), 'List of issues found'),
+    suggestions: field.array(z.string(), 'Improvement suggestions'),
+    score: field.number('Code quality score 0-100'),
+  },
+})
+```
+
+**Field helpers:**
+- `field.string(desc?)` - String field
+- `field.number(desc?)` - Number field
+- `field.boolean(desc?)` - Boolean field
+- `field.array(itemType, desc?)` - Array field
+- `field.object(shape, desc?)` - Object field
+- `field.enum(values, desc?)` - Enum field
+- `field.optional(field)` - Optional wrapper
+- `field.nullable(field)` - Nullable wrapper
+- `field.custom(zodType, desc?)` - Custom Zod type
+
+### Predict
+
+`Predict` is the bridge between a Signature (the contract) and OpenCode (the execution). It handles prompt generation, response parsing, and validation.
+
+```typescript
+import { Predict } from 'dsts'
+
+// Basic usage
+const predict = new Predict(AnalyzeCode)
+const result = await predict.execute({ code: '...', language: 'typescript' }, ctx)
+
+// With configuration
+const predict = new Predict(AnalyzeCode, {
+  agent: 'code-reviewer',      // Override default agent
+  model: { providerID: 'anthropic', modelID: 'claude-opus-4-5' },
+  newSession: true,            // Don't reuse existing session
+  template: (inputs) => `...`, // Custom prompt template
+})
+```
+
+**Output format:**
+
+The LLM is prompted to return a JSON object:
+```
+OUTPUT FORMAT:
+Return a JSON object with EXACTLY these field names and types.
+
+```json
+{
+  "issues": <array<object{severity, message, line}>>,  // List of issues found
+  "suggestions": <array<string>>,  // Improvement suggestions
+  "score": <number>  // Code quality score 0-100
+}
+```
+```
+
+### Module
+
+A Module encapsulates a logical unit of work that may use one or more Predictors. Modules can call other Modules, enabling composition.
+
+**SignatureModule** - For simple modules that wrap a single signature with pass-through types:
+
+```typescript
+import { SignatureModule } from 'dsts'
+
+class IntentParser extends SignatureModule<typeof ParseIntent> {
+  constructor() {
+    super(ParseIntent)
+  }
+
+  async forward(input, ctx) {
+    const result = await this.predictor.execute(input, ctx)
+    return result.data  // Types inferred from ParseIntent
+  }
+}
+```
+
+**Module** - For complex modules with multiple predictors or transformed outputs:
+
+```typescript
+import { Module } from 'dsts'
+
+class CodeAnalyzer extends Module<
+  { code: string; language: string },
+  { issues: Issue[]; score: number }
+> {
+  private analyze = this.predict(AnalyzeCode)
+  private suggest = this.predict(SuggestFixes, { agent: 'code-fixer' })
+
+  async forward(input: { code: string; language: string }, ctx: ExecutionContext) {
+    // First, analyze the code
+    const analysis = await this.analyze.execute(input, ctx)
+    
+    // If there are critical issues, get fix suggestions
+    if (analysis.data.issues.some(i => i.severity === 'error')) {
+      const fixes = await this.suggest.execute({
+        code: input.code,
+        issues: analysis.data.issues,
+      }, ctx)
+      
+      return {
+        issues: analysis.data.issues,
+        fixes: fixes.data.suggestions,
+        score: analysis.data.score,
+      }
+    }
+
+    return {
+      issues: analysis.data.issues,
+      score: analysis.data.score,
+    }
+  }
+}
+```
+
+### Pipeline
+
+Pipeline is the top-level orchestrator. It manages execution context, state, checkpointing, logging, and retry logic.
+
+```typescript
+import { Pipeline, createBaseState } from 'dsts'
+
+// Create pipeline with configuration
+const pipeline = new Pipeline({
+  name: 'code-review',
+  defaultModel: { providerID: 'anthropic', modelID: 'claude-sonnet-4-5' },
+  defaultAgent: 'general',
+  checkpointDir: './ckpt',
+  logDir: './logs',
+  retry: { maxAttempts: 2, onParseError: true },
+  timeoutSec: 300,
+}, createBaseState)
+
+// Run modules
+const result = await pipeline.run(new CodeAnalyzer(), {
+  code: sourceCode,
+  language: 'typescript',
+})
+
+// Run with step options
+const result = await pipeline.run(new CodeAnalyzer(), input, {
+  name: 'analyze-main',        // Custom step name
+  model: { providerID: 'anthropic', modelID: 'claude-opus-4-5' },  // Override model
+  newSession: true,            // Fresh session
+  retry: { maxAttempts: 3 },   // Override retry
+})
+
+// Access state
+console.log(pipeline.state.steps)        // Completed steps
+console.log(pipeline.getSessionId())     // Current OpenCode session
+
+// Resume from checkpoint
+const resumed = await Pipeline.loadCheckpoint(config, sessionId)
+```
+
+### State Management
+
+DSTS automatically checkpoints state after each step:
+
+```typescript
+import { createBaseState, extendBaseState } from 'dsts'
+
+// Basic state
+const state = createBaseState()
+// { sessionId, startedAt, phase, steps, subPipelines }
+
+// Extended state for your workflow
+interface MyState extends BaseState {
+  inputPath: string
+  results: AnalysisResult[]
+}
+
+const pipeline = new Pipeline(config, () => ({
+  ...createBaseState(),
+  inputPath: '/path/to/input',
+  results: [],
+}))
+```
+
+## Testing
+
+DSTS provides testing utilities for unit testing without hitting real LLMs:
+
+```typescript
+import { MockAgentBackend, createMockContext, generateMockOutputs } from 'dsts'
+import { vi } from 'vitest'
+
+// Create mock backend
+const mock = new MockAgentBackend()
+
+// Add mock responses
+mock.addJsonResponse({
+  intent: 'greeting',
+  confidence: 0.95,
+  keywords: ['hello', 'world'],
+})
+
+// Mock the agent module
+vi.mock('./agent.js', () => ({
+  runAgent: mock.createRunner(),
+}))
+
+// Create test context
+const ctx = createMockContext({
+  defaultModel: { providerID: 'anthropic', modelID: 'claude-sonnet-4-5' },
+})
+
+// Auto-generate mock outputs from schema
+const mockData = generateMockOutputs(ParseIntent.outputs)
+```
+
+## Why Not ChainOfThought or ReAct?
+
+Unlike DSPy, DSTS does not provide `ChainOfThought` or `ReAct` variants. This is intentional:
+
+- **OpenCode agents already do chain-of-thought reasoning** - they think before acting
+- **OpenCode agents already do ReAct** - they have access to tools and use them iteratively
+- **Adding these would duplicate functionality** and create confusion
+
+If you need tool access, configure your OpenCode agent appropriately. The agent handles the complexity; DSTS just structures the input/output contract.
+
+## Requirements
+
+- [Bun](https://bun.sh) runtime
+- [OpenCode](https://opencode.ai) CLI installed and configured
+- [Zod](https://zod.dev) for schema validation
+
+## Installation
+
+```bash
+bun add dsts zod
+```
+
+## License
+
+MIT - see [LICENSE](https://github.com/s4wave/dsts/blob/main/LICENSE) for details.

package/agent.ts

@@ -0,0 +1,176 @@
+/**
+ * DSTS SDK OpenCode agent integration.
+ *
+ * Wraps the OpenCode CLI for running LLM agents with session management.
+ */
+
+import { spawn } from 'child_process'
+import { mkdir } from 'fs/promises'
+import { PROJECT_ROOT, TMP_DIR } from '../paths.js'
+import type { RunAgentOptions, RunAgentResult } from './types.js'
+
+/** runAgent executes an OpenCode agent with a prompt, streaming output in real-time. */
+export async function runAgent(
+  options: RunAgentOptions,
+): Promise<RunAgentResult> {
+  const { prompt, agent, model, sessionId, timeoutSec = 300 } = options
+
+  const modelStr = `${model.providerID}/${model.modelID}`
+  const sessionInfo = sessionId ? `[session:${sessionId}]` : '[new session]'
+  const promptPreview = prompt.slice(0, 50).replace(/\n/g, ' ')
+
+  console.error(
+    `\n>>> OpenCode [${agent}] [${modelStr}] ${sessionInfo}: ${promptPreview}...`,
+  )
+
+  const args = ['run', '--format', 'default', '--agent', agent, '--model', modelStr]
+
+  if (sessionId) {
+    args.push('--session', sessionId)
+  }
+
+  return new Promise((resolve, reject) => {
+    const proc = spawn('opencode', args, {
+      cwd: PROJECT_ROOT,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    })
+
+    let newSessionId = sessionId || ''
+    const stdoutChunks: string[] = []
+
+    // Stream stderr in real-time (OpenCode progress output)
+    proc.stderr.on('data', (data: Buffer) => {
+      const text = data.toString()
+
+      // Parse session ID from output
+      for (const line of text.split('\n')) {
+        if (line.startsWith('[session:')) {
+          newSessionId = line.trim().slice(9, -1)
+          continue
+        }
+        // Filter noise
+        if (line.includes('baseline-browser-mapping')) continue
+        if (line.startsWith('$ bun run')) continue
+        if (line.trim()) {
+          process.stderr.write(line + '\n')
+        }
+      }
+    })
+
+    // Collect stdout
+    proc.stdout.on('data', (data: Buffer) => {
+      const text = data.toString()
+      stdoutChunks.push(text)
+      process.stderr.write(text)
+    })
+
+    // Send prompt to stdin
+    proc.stdin.write(prompt)
+    proc.stdin.end()
+
+    // Timeout handling
+    const timeout = setTimeout(() => {
+      proc.kill()
+      reject(new Error(`Timeout after ${timeoutSec}s`))
+    }, timeoutSec * 1000)
+
+    proc.on('close', async (code) => {
+      clearTimeout(timeout)
+
+      if (code !== 0) {
+        reject(new Error(`OpenCode exited with code ${code}`))
+        return
+      }
+
+      // Export session to get structured response
+      let response = stdoutChunks.join('').trim()
+
+      if (newSessionId) {
+        const exported = await exportSession(newSessionId)
+        if (exported) {
+          response = exported
+        }
+      }
+
+      const sessionStr = newSessionId || 'none'
+      console.error(
+        `<<< OpenCode done (${response.length} chars) [session:${sessionStr}]`,
+      )
+
+      resolve({
+        text: response,
+        sessionId: newSessionId,
+      })
+    })
+
+    proc.on('error', (err) => {
+      clearTimeout(timeout)
+      reject(err)
+    })
+  })
+}
+
+/** exportSession exports a session and extracts assistant text responses. */
+async function exportSession(sessionId: string): Promise<string | null> {
+  const tmpPath = `${TMP_DIR}/opencode_export_${Date.now()}.json`
+
+  try {
+    await mkdir(TMP_DIR, { recursive: true })
+    const proc = Bun.spawn(
+      [
+        'opencode',
+        'session',
+        'export',
+        sessionId,
+        '--format',
+        'json',
+        '-o',
+        tmpPath,
+      ],
+      {
+        cwd: PROJECT_ROOT,
+        stdout: 'pipe',
+        stderr: 'pipe',
+      },
+    )
+
+    await proc.exited
+
+    const file = Bun.file(tmpPath)
+    if (!(await file.exists())) return null
+
+    const data = (await file.json()) as {
+      messages?: Array<{
+        info?: { role?: string }
+        parts?: Array<{ type?: string; text?: string }>
+      }>
+    }
+    await Bun.write(tmpPath, '') // Clean up
+
+    // Extract all assistant text parts
+    const messages = data.messages || []
+    const textParts: string[] = []
+
+    for (const msg of messages) {
+      if (msg.info?.role === 'assistant') {
+        for (const part of msg.parts || []) {
+          if (part.type === 'text' && part.text) {
+            textParts.push(part.text)
+          }
+        }
+      }
+    }
+
+    return textParts.length > 0 ? textParts.join('\n') : null
+  } catch {
+    return null
+  }
+}
+
+/** logStep logs a step header for workflow progress. */
+export function logStep(step: number, title: string, detail = ''): void {
+  const detailStr = detail ? ` (${detail})` : ''
+  console.log(`\n${'='.repeat(60)}`)
+  console.log(`STEP ${step}: ${title}${detailStr}`)
+  console.log('='.repeat(60))
+}

package/example/correction.ts

@@ -0,0 +1,85 @@
+/**
+ * Auto-correction example.
+ *
+ * Demonstrates DSTS's automatic schema correction.
+ * This example uses a schema with specific field names that LLMs
+ * sometimes get wrong (e.g., "type" instead of "issue_type").
+ *
+ * DSTS supports two correction methods:
+ * - 'json-patch' (default): RFC 6902 JSON Patch, no external dependencies
+ * - 'jq': jq-style expressions, requires jq binary installed
+ *
+ * Usage:
+ *   bun run example/correction.ts              # Uses default (json-patch)
+ *   bun run example/correction.ts --jq         # Uses jq method
+ */
+
+import { z } from 'zod'
+import { Pipeline, createBaseState, signature, field, SignatureModule } from '../index.js'
+import type { CorrectionMethod, ExecutionContext } from '../types.js'
+
+// A signature with field names that LLMs often get wrong
+const AnalyzeIssue = signature({
+  doc: `Analyze the given code issue and categorize it.
+
+IMPORTANT: Use the EXACT field names specified in the schema.`,
+  inputs: {
+    description: field.string('Description of the code issue'),
+  },
+  outputs: {
+    // LLMs often return "type" instead of "issue_type"
+    issue_type: field.enum(['bug', 'feature', 'refactor', 'docs'] as const, 'Category of the issue'),
+    // LLMs often return "priority" instead of "severity"
+    severity: field.enum(['low', 'medium', 'high', 'critical'] as const, 'How severe is the issue'),
+    // LLMs often return "description" or "reason" instead of "explanation"
+    explanation: field.string('Detailed explanation of the issue'),
+    // LLMs often return just "tags" or "labels"
+    suggested_tags: field.array(z.string(), 'Tags to apply to this issue'),
+  },
+})
+
+class IssueAnalyzer extends SignatureModule<typeof AnalyzeIssue> {
+  constructor(method: CorrectionMethod = 'json-patch') {
+    super(AnalyzeIssue, {
+      correction: { method },
+    })
+  }
+
+  async forward(input: { description: string }, ctx: ExecutionContext) {
+    const result = await this.predictor.execute(input, ctx)
+    return result.data
+  }
+}
+
+async function main() {
+  // Check for --jq flag
+  const method: CorrectionMethod = process.argv.includes('--jq') ? 'jq' : 'json-patch'
+
+  const pipeline = new Pipeline(
+    {
+      name: 'correction-demo',
+      defaultModel: { providerID: 'anthropic', modelID: 'claude-haiku-4-5' },
+      defaultAgent: 'code',
+      checkpointDir: './ckpt',
+      logDir: './logs',
+    },
+    createBaseState,
+  )
+
+  console.log('=== Auto-Correction Demo ===')
+  console.log(`Correction method: ${method}`)
+  console.log('This example uses field names that LLMs often get wrong.')
+  console.log('Watch the correction rounds fix schema mismatches.\n')
+
+  const result = await pipeline.run(new IssueAnalyzer(method), {
+    description: 'The login button does not respond when clicked on mobile devices',
+  })
+
+  console.log('\n=== Final Result ===')
+  console.log(`Issue Type: ${result.data.issue_type}`)
+  console.log(`Severity: ${result.data.severity}`)
+  console.log(`Explanation: ${result.data.explanation}`)
+  console.log(`Tags: ${result.data.suggested_tags.join(', ')}`)
+}
+
+main().catch(console.error)

package/example/index.ts

@@ -0,0 +1,31 @@
+/**
+ * Hello World example runner.
+ *
+ * Demonstrates running a DSTS module in a pipeline.
+ */
+
+import { Pipeline, createBaseState } from '../index.js'
+import { Greeter } from './module.js'
+
+async function main() {
+  // Create a pipeline with configuration
+  const pipeline = new Pipeline(
+    {
+      name: 'hello-world',
+      defaultModel: { providerID: 'anthropic', modelID: 'claude-haiku-4-5' },
+      defaultAgent: 'code',
+      checkpointDir: './ckpt',
+      logDir: './logs',
+    },
+    createBaseState,
+  )
+
+  // Run the greeter module
+  const result = await pipeline.run(new Greeter(), { name: 'World' })
+
+  console.log('\n=== Result ===')
+  console.log(`Greeting: ${result.data.greeting}`)
+  console.log(`Emoji: ${result.data.emoji}`)
+}
+
+main().catch(console.error)

package/example/module.ts

@@ -0,0 +1,20 @@
+/**
+ * Hello World module.
+ *
+ * Wraps the Greet signature with execution logic.
+ */
+
+import { SignatureModule } from '../index.js'
+import type { ExecutionContext } from '../types.js'
+import { Greet } from './signature.js'
+
+export class Greeter extends SignatureModule<typeof Greet> {
+  constructor() {
+    super(Greet)
+  }
+
+  async forward(input: { name: string }, ctx: ExecutionContext) {
+    const result = await this.predictor.execute(input, ctx)
+    return result.data
+  }
+}

package/example/signature.ts

@@ -0,0 +1,18 @@
+/**
+ * Hello World signature.
+ *
+ * Defines the input/output contract for greeting generation.
+ */
+
+import { signature, field } from '../index.js'
+
+export const Greet = signature({
+  doc: 'Generate a friendly greeting for the given name.',
+  inputs: {
+    name: field.string('The name of the person to greet'),
+  },
+  outputs: {
+    greeting: field.string('A friendly greeting message'),
+    emoji: field.string('An appropriate emoji for the greeting'),
+  },
+})