@studio-foundation/engine 0.3.0-beta.1

package/README.md

@@ -0,0 +1,118 @@
+# @studio-foundation/engine
+
+Pipeline orchestration, state machine, persistence, hooks, and skills injection.
+
+## Role
+
+engine is the conductor. It loads pipeline configs, sequences stages, delegates execution to ralph+runner, tracks state in SQLite, and emits events for observability. It knows about pipelines, stages, groups, hooks, and skills — but never about LLMs, files, or domain concepts.
+
+```
+cli → engine.run(pipeline, input) → PipelineRun
+          ↓
+      [load pipeline] → [on_pipeline_start] → [for each stage: hooks + ralph(runner)] → [persist state] → [emit events]
+```
+
+## Key exports
+
+```typescript
+import { PipelineEngine } from '@studio-foundation/engine';
+
+const engine = new PipelineEngine({
+  configsDir: '.studio',          // Root of .studio/ directory
+  repoPath: '/path/to/workspace', // Where tools operate (optional)
+  providerRegistry,
+  toolRegistry,
+  db: runStore,
+  pluginSkills: {                 // Skills from Claude Code plugins, keyed by plugin name
+    'my-plugin': ['## Skill: commit-conventions\n...'],
+  },
+});
+
+const run = await engine.run({
+  pipeline: 'feature-builder',
+  input: { brief: 'Add dark mode' },
+  anonymize: true,  // Enable PII anonymization for this run
+});
+```
+
+## State machine
+
+```
+pending → running → success
+                  → failed     (ralph exhausted all attempts, or on_stage_start hook with on_failure: fail)
+                  → rejected   (post_validation rejection, or hook with on_failure: reject)
+                  → skipped
+```
+
+`deriveStageStatus()` in `state/status-derivation.ts` is the critical mapping function: ralph result → stage status. One place, deterministic.
+
+## Lifecycle hooks
+
+The engine executes stage hooks at four deterministic points:
+
+| Hook | When | Template vars |
+|------|------|---------------|
+| `on_stage_start` | Before ralph loop | None |
+| `on_stage_complete` | After stage succeeds | `{{output.field}}` |
+| `pre_tool_use` | Before a tool call (matcher-gated) | `{{tool.argName}}` |
+| `post_tool_use` | After a tool call (matcher-gated) | `{{tool.argName}}` |
+
+Hook failure semantics via `on_failure`:
+- `warn` (default) — log and continue
+- `reject` — stage → `rejected` (can trigger group retry)
+- `fail` — stage → `failed` (stops pipeline)
+
+`pre_tool_use` hooks with any failure block the tool call. Hook commands run in `repoPath` (or `configsDir` as fallback). Implemented in `pipeline/hook-executor.ts`.
+
+## on_pipeline_start
+
+Commands in `pipeline.on_pipeline_start` execute before any stage and inject their stdout into the pipeline context:
+
+```yaml
+on_pipeline_start:
+  - command: "git status --short"
+    inject_as: git_status
+```
+
+Implemented in `pipeline/startup-executor.ts`. Failures are non-fatal (logged, pipeline continues).
+
+## Skills injection
+
+Agents that declare `skills: [name]` in their YAML get `.studio/skills/<name>.skill.md` auto-injected into their system prompt. Loaded by `pipeline/skill-loader.ts`. Missing skill files are non-fatal (warned, skipped).
+
+Plugin skills (`pluginSkills` in `EngineConfig`) are injected for agents that declare `plugins: [plugin-name]`.
+
+## Events
+
+The engine emits structured events at every lifecycle point. See `events.ts`:
+
+| Event | When |
+|-------|------|
+| `onPipelineStart` / `onPipelineComplete` | Pipeline lifecycle |
+| `onStageStart` / `onStageComplete` | Stage lifecycle |
+| `onTaskRetry` | RALPH retry |
+| `onGroupStart` / `onGroupIteration` / `onGroupFeedback` / `onGroupComplete` | Group lifecycle |
+| `onToolCallStart` / `onToolCallComplete` | Tool call lifecycle |
+| `onAgentThinking` / `onAgentProgress` / `onAgentToken` | Streaming events |
+
+The CLI subscribes to these events to render progress output and stream tokens.
+
+## Groups
+
+A group is a set of stages that iterate together. If the last stage rejects (via `post_validation.rejection_detection` in its contract), the group reruns from the first stage with accumulated feedback. `max_iterations` caps the loop.
+
+## PII Anonymization
+
+```typescript
+engine.run({ pipeline, input, anonymize: true })
+// Keymap persisted to .studio/runs/anonymization/<run-id>.keymap.json
+```
+
+Per-agent anonymization also supported via `anonymize: true` in agent YAML.
+
+## Rules
+
+- **engine is domain-agnostic.** No references to "code", "file", "git", "QA" in engine source. All domain knowledge is in YAML configs.
+- **engine doesn't execute tools.** It passes tool configs to runner. The runner decides what `repo_manager-write_file` means.
+- **engine doesn't build prompts.** That's runner's job.
+- Persistence: `PgRunStore` (PostgreSQL) for production, `InMemoryRunStore` for tests. Both implement `AnyRunStore`.

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@studio-foundation/engine",
+  "version": "0.3.0-beta.1",
+  "description": "Pipeline orchestration engine for Studio v7",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "keywords": [
+    "studio",
+    "engine",
+    "pipeline",
+    "orchestration"
+  ],
+  "author": "Ariane Guay",
+  "license": "AGPL-3.0-only",
+  "dependencies": {
+    "better-sqlite3": "^9.0.0",
+    "js-yaml": "^4.1.0",
+    "pg": "^8.19.0",
+    "@studio-foundation/contracts": "0.3.0-beta.1",
+    "@studio-foundation/ralph": "0.3.0-beta.1",
+    "@studio-foundation/runner": "0.3.0-beta.1"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.8",
+    "@types/js-yaml": "^4.0.9",
+    "@types/pg": "^8.18.0",
+    "@types/node": "^25.2.3",
+    "typescript": "^5.3.0",
+    "vitest": "^4.0.18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest --coverage"
+  }
+}

package/src/__tests__/__fixtures__/script-stage/contracts/book-context.contract.yaml

@@ -0,0 +1,4 @@
+name: book-context
+version: 1
+schema:
+  required_fields: []

package/src/__tests__/engine.conditions-group.test.ts

@@ -0,0 +1,244 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { resolve } from 'node:path';
+import { PipelineEngine } from '../engine.js';
+import type { PipelineDefinition } from '@studio-foundation/contracts';
+
+vi.mock('@studio-foundation/runner', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@studio-foundation/runner')>();
+  return {
+    ...actual,
+    runScript: vi.fn(),
+  };
+});
+
+import { runScript } from '@studio-foundation/runner';
+
+const FIXTURES_DIR = resolve(__dirname, '__fixtures__/script-stage');
+
+function makeEngine() {
+  return new PipelineEngine({
+    configsDir: FIXTURES_DIR,
+    providerRegistry: {} as any,
+  });
+}
+
+describe('engine — conditions inside sequential groups', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('skips a conditional stage inside a sequential group', async () => {
+    vi.mocked(runScript).mockResolvedValue({
+      output: { done: true },
+      tool_calls: [],
+      tool_calls_count: 0,
+      duration_ms: 10,
+    });
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-group-conditions',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          group: 'processing',
+          max_iterations: 1,
+          stages: [
+            {
+              name: 'always-runs',
+              executor: 'script',
+              script: 'scripts/run.py',
+              runtime: 'shell',
+            },
+            {
+              name: 'conditional-in-group',
+              executor: 'script',
+              script: 'scripts/optional.py',
+              runtime: 'shell',
+              condition: 'input.optional >= 1',
+            },
+          ],
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { optional: 0 },
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('success');
+    expect(result.stages[1]?.status).toBe('skipped');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);
+  });
+
+  it('marks group as skipped when all stages in sequential group are skipped', async () => {
+    vi.mocked(runScript).mockResolvedValue({
+      output: { done: true },
+      tool_calls: [],
+      tool_calls_count: 0,
+      duration_ms: 10,
+    });
+
+    // A stage after the group should still run
+    const pipeline: PipelineDefinition = {
+      name: 'test-all-skipped-group',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          group: 'optional-processing',
+          max_iterations: 1,
+          stages: [
+            {
+              name: 'optional-a',
+              executor: 'script',
+              script: 'scripts/a.py',
+              runtime: 'shell',
+              condition: 'input.run_optional >= 1',
+            },
+            {
+              name: 'optional-b',
+              executor: 'script',
+              script: 'scripts/b.py',
+              runtime: 'shell',
+              condition: 'input.run_optional >= 1',
+            },
+          ],
+        },
+        {
+          name: 'post-group-stage',
+          executor: 'script',
+          script: 'scripts/final.py',
+          runtime: 'shell',
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { run_optional: 0 },
+    });
+
+    expect(result.status).toBe('success');
+    // Both group stages skipped
+    expect(result.stages[0]?.status).toBe('skipped');
+    expect(result.stages[1]?.status).toBe('skipped');
+    // Post-group stage ran
+    expect(result.stages[2]?.status).toBe('success');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);  // only post-group-stage
+  });
+});
+
+describe('engine — conditions inside parallel groups', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('marks parallel group as skipped when all stages are skipped', async () => {
+    vi.mocked(runScript).mockResolvedValue({
+      output: { done: true },
+      tool_calls: [],
+      tool_calls_count: 0,
+      duration_ms: 10,
+    });
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-parallel-all-skipped',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          group: 'optional-parallel',
+          max_iterations: 1,
+          mode: 'parallel',
+          stages: [
+            {
+              name: 'parallel-a',
+              executor: 'script',
+              script: 'scripts/a.py',
+              runtime: 'shell',
+              condition: 'input.enabled >= 1',
+            },
+            {
+              name: 'parallel-b',
+              executor: 'script',
+              script: 'scripts/b.py',
+              runtime: 'shell',
+              condition: 'input.enabled >= 1',
+            },
+          ],
+        },
+        {
+          name: 'post-group',
+          executor: 'script',
+          script: 'scripts/final.py',
+          runtime: 'shell',
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { enabled: 0 },
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('skipped');
+    expect(result.stages[1]?.status).toBe('skipped');
+    expect(result.stages[2]?.status).toBe('success');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);  // only post-group
+  });
+
+  it('runs parallel group normally when at least one stage is not skipped', async () => {
+    vi.mocked(runScript).mockResolvedValue({
+      output: { done: true },
+      tool_calls: [],
+      tool_calls_count: 0,
+      duration_ms: 10,
+    });
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-parallel-partial-skip',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          group: 'mixed-parallel',
+          max_iterations: 1,
+          mode: 'parallel',
+          stages: [
+            {
+              name: 'always-runs',
+              executor: 'script',
+              script: 'scripts/a.py',
+              runtime: 'shell',
+            },
+            {
+              name: 'conditional',
+              executor: 'script',
+              script: 'scripts/b.py',
+              runtime: 'shell',
+              condition: 'input.enabled >= 1',
+            },
+          ],
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { enabled: 0 },
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('success');
+    expect(result.stages[1]?.status).toBe('skipped');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);  // only always-runs
+  });
+});

package/src/__tests__/engine.conditions.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { resolve } from 'node:path';
+import { PipelineEngine } from '../engine.js';
+import type { PipelineDefinition } from '@studio-foundation/contracts';
+
+// Mock the runner so we can inspect which stages actually ran
+vi.mock('@studio-foundation/runner', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@studio-foundation/runner')>();
+  return {
+    ...actual,
+    runScript: vi.fn(),
+  };
+});
+
+import { runScript } from '@studio-foundation/runner';
+
+const FIXTURES_DIR = resolve(__dirname, '__fixtures__/script-stage');
+
+function makeEngine() {
+  return new PipelineEngine({
+    configsDir: FIXTURES_DIR,
+    providerRegistry: {} as any,
+  });
+}
+
+function mockScriptSuccess(output: Record<string, unknown> = { result: 'ok' }) {
+  vi.mocked(runScript).mockResolvedValue({
+    output,
+    tool_calls: [],
+    tool_calls_count: 0,
+    duration_ms: 10,
+  });
+}
+
+describe('engine — stage conditions', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('skips a stage when input condition is false', async () => {
+    mockScriptSuccess();
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-conditions',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          name: 'always-runs',
+          executor: 'script',
+          script: 'scripts/parse.py',
+          runtime: 'shell',
+        },
+        {
+          name: 'conditional-stage',
+          executor: 'script',
+          script: 'scripts/parse.py',
+          runtime: 'shell',
+          condition: 'input.meals_count >= 6',
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { meals_count: 3 },  // condition is false — stage should skip
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages).toHaveLength(2);
+    expect(result.stages[0]?.status).toBe('success');
+    expect(result.stages[1]?.status).toBe('skipped');
+    // Script was only called once (for always-runs, not conditional-stage)
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);
+  });
+
+  it('runs a stage when input condition is true', async () => {
+    mockScriptSuccess();
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-conditions-true',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          name: 'conditional-stage',
+          executor: 'script',
+          script: 'scripts/parse.py',
+          runtime: 'shell',
+          condition: 'input.meals_count >= 6',
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { meals_count: 7 },  // condition is true — stage should run
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('success');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);
+  });
+
+  it('skips a stage based on previous stage output', async () => {
+    // First call returns extraction result, second call would be entity-resolution
+    vi.mocked(runScript).mockResolvedValueOnce({
+      output: { counts: { OTHER: 0, PERSON: 2 } },
+      tool_calls: [],
+      tool_calls_count: 0,
+      duration_ms: 10,
+    });
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-conditions-stage-output',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          name: 'entity-extraction',
+          executor: 'script',
+          script: 'scripts/extract.py',
+          runtime: 'shell',
+          context: { include: ['input'] },
+        },
+        {
+          name: 'entity-resolution-OTHER',
+          executor: 'script',
+          script: 'scripts/resolve.py',
+          runtime: 'shell',
+          condition: 'stages.entity-extraction.output.counts.OTHER > 0',
+          context: { include: ['all_stage_outputs'] },
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: 'extract entities',
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('success');
+    expect(result.stages[1]?.status).toBe('skipped');  // counts.OTHER is 0
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);  // only extraction ran
+  });
+
+  it('pipeline continues after skipped stage', async () => {
+    mockScriptSuccess({ final: 'result' });
+
+    const pipeline: PipelineDefinition = {
+      name: 'test-skip-continues',
+      description: 'test',
+      version: 1,
+      stages: [
+        {
+          name: 'skipped-stage',
+          executor: 'script',
+          script: 'scripts/parse.py',
+          runtime: 'shell',
+          condition: 'input.run_optional >= 1',
+        },
+        {
+          name: 'final-stage',
+          executor: 'script',
+          script: 'scripts/finalize.py',
+          runtime: 'shell',
+        },
+      ],
+    };
+
+    const engine = makeEngine();
+    const result = await engine.run({
+      pipelineDef: pipeline,
+      input: { run_optional: 0 },
+    });
+
+    expect(result.status).toBe('success');
+    expect(result.stages[0]?.status).toBe('skipped');
+    expect(result.stages[1]?.status).toBe('success');
+    expect(vi.mocked(runScript)).toHaveBeenCalledTimes(1);  // only final-stage
+  });
+});