@studio-foundation/engine 0.3.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +57 -0
- package/LICENSE +663 -0
- package/README.md +118 -0
- package/package.json +47 -0
- package/src/__tests__/__fixtures__/script-stage/contracts/book-context.contract.yaml +4 -0
- package/src/__tests__/engine.conditions-group.test.ts +244 -0
- package/src/__tests__/engine.conditions.test.ts +186 -0
- package/src/__tests__/engine.resume.test.ts +108 -0
- package/src/__tests__/engine.script-stage.test.ts +125 -0
- package/src/db/client.ts +5 -0
- package/src/engine.context-event.test.ts +175 -0
- package/src/engine.ts +491 -0
- package/src/events.ts +167 -0
- package/src/index.ts +64 -0
- package/src/pipeline/agent-loader.test.ts +151 -0
- package/src/pipeline/agent-loader.ts +39 -0
- package/src/pipeline/condition-evaluator.test.ts +129 -0
- package/src/pipeline/condition-evaluator.ts +121 -0
- package/src/pipeline/context-pack-loader.ts +63 -0
- package/src/pipeline/context-propagation.test.ts +237 -0
- package/src/pipeline/context-propagation.ts +235 -0
- package/src/pipeline/contract-loader.ts +41 -0
- package/src/pipeline/group-orchestrator.ts +483 -0
- package/src/pipeline/hook-executor.test.ts +121 -0
- package/src/pipeline/hook-executor.ts +87 -0
- package/src/pipeline/invariants-loader.test.ts +51 -0
- package/src/pipeline/invariants-loader.ts +14 -0
- package/src/pipeline/loader.test.ts +128 -0
- package/src/pipeline/loader.ts +149 -0
- package/src/pipeline/output-validator.test.ts +100 -0
- package/src/pipeline/output-validator.ts +40 -0
- package/src/pipeline/post-validator.ts +124 -0
- package/src/pipeline/skill-loader.test.ts +44 -0
- package/src/pipeline/skill-loader.ts +32 -0
- package/src/pipeline/stage-executor.ts +654 -0
- package/src/pipeline/stage-resolver.ts +8 -0
- package/src/pipeline/startup-executor.test.ts +37 -0
- package/src/pipeline/startup-executor.ts +32 -0
- package/src/pipeline/types.ts +42 -0
- package/src/repo-resolver.ts +61 -0
- package/src/spawners/direct-engine-spawner.ts +26 -0
- package/src/state/run-store.test.ts +157 -0
- package/src/state/run-store.ts +362 -0
- package/src/state/state-machine.ts +35 -0
- package/src/state/status-derivation.ts +36 -0
- package/tests/context-pack-loader.test.ts +113 -0
- package/tests/context-propagation.test.ts +267 -0
- package/tests/direct-engine-spawner.test.ts +102 -0
- package/tests/e2e/feature-v5.test.ts +57 -0
- package/tests/events.test.ts +56 -0
- package/tests/fixtures/agents/test-agent.agent.yaml +5 -0
- package/tests/fixtures/contracts/code-gen.contract.yaml +6 -0
- package/tests/fixtures/contracts/qa-gate.contract.yaml +15 -0
- package/tests/fixtures/contracts/test-contract.contract.yaml +6 -0
- package/tests/fixtures/pipelines/group-test.pipeline.yaml +40 -0
- package/tests/fixtures/pipelines/simple.pipeline.yaml +15 -0
- package/tests/fixtures/pipelines/two-stage.pipeline.yaml +24 -0
- package/tests/fixtures/software/pipelines/feature-builder.pipeline.yaml +75 -0
- package/tests/fixtures/test-project/agents/test-agent.agent.yaml +5 -0
- package/tests/fixtures/test-project/contracts/basic-result.contract.yaml +6 -0
- package/tests/fixtures/test-project/contracts/code-gen.contract.yaml +6 -0
- package/tests/fixtures/test-project/contracts/maximum-only.contract.yaml +8 -0
- package/tests/fixtures/test-project/contracts/qa-gate.contract.yaml +15 -0
- package/tests/fixtures/test-project/contracts/strict-result.contract.yaml +7 -0
- package/tests/fixtures/test-project/contracts/test-contract.contract.yaml +6 -0
- package/tests/fixtures/test-project/pipelines/group-simple.pipeline.yaml +26 -0
- package/tests/fixtures/test-project/pipelines/group-test.pipeline.yaml +40 -0
- package/tests/fixtures/test-project/pipelines/hook-output-template.pipeline.yaml +19 -0
- package/tests/fixtures/test-project/pipelines/hook-reject-on-failure.pipeline.yaml +19 -0
- package/tests/fixtures/test-project/pipelines/maximum-only.pipeline.yaml +15 -0
- package/tests/fixtures/test-project/pipelines/parallel-collect-all-test.pipeline.yaml +39 -0
- package/tests/fixtures/test-project/pipelines/parallel-context-isolation-test.pipeline.yaml +38 -0
- package/tests/fixtures/test-project/pipelines/parallel-fail-test.pipeline.yaml +39 -0
- package/tests/fixtures/test-project/pipelines/parallel-test.pipeline.yaml +39 -0
- package/tests/fixtures/test-project/pipelines/parallel-then-sequential-test.pipeline.yaml +38 -0
- package/tests/fixtures/test-project/pipelines/simple.pipeline.yaml +15 -0
- package/tests/fixtures/test-project/pipelines/two-stage.pipeline.yaml +24 -0
- package/tests/fixtures/test-project/pipelines/with-startup.pipeline.yaml +19 -0
- package/tests/loader.test.ts +385 -0
- package/tests/post-validator.test.ts +297 -0
- package/tests/repo-resolver.test.ts +102 -0
- package/tests/run-store.test.ts +143 -0
- package/tests/state-machine.test.ts +110 -0
- package/tests/unit/context-propagation.test.ts +45 -0
- package/tests/unit/engine.test.ts +770 -0
- package/tests/unit/group-loop.test.ts +389 -0
- package/tests/unit/group-parallel.test.ts +423 -0
- package/tests/unit/state/status-derivation.test.ts +88 -0
- package/tsconfig.json +24 -0
- package/vitest.config.ts +14 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
2
|
+
import { mkdir, writeFile, rm } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { loadInvariantsFile } from './invariants-loader.js';
|
|
5
|
+
|
|
6
|
+
const TMP = join('/tmp', '.studio-invariants-loader-test-' + Date.now());
|
|
7
|
+
const INVARIANTS_PATH = join(TMP, 'invariants.md');
|
|
8
|
+
|
|
9
|
+
describe('loadInvariantsFile', () => {
|
|
10
|
+
beforeAll(async () => {
|
|
11
|
+
await mkdir(TMP, { recursive: true });
|
|
12
|
+
await writeFile(INVARIANTS_PATH, '# Domain Invariants\n\nNever reproduce verbatim passages.');
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
afterAll(async () => {
|
|
16
|
+
await rm(TMP, { recursive: true, force: true });
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('returns file content when invariants.md exists', async () => {
|
|
20
|
+
const content = await loadInvariantsFile(TMP);
|
|
21
|
+
expect(content).toBe('# Domain Invariants\n\nNever reproduce verbatim passages.');
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it('returns undefined when invariants.md does not exist', async () => {
|
|
25
|
+
const content = await loadInvariantsFile('/tmp/no-such-studio-dir-xyz');
|
|
26
|
+
expect(content).toBeUndefined();
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
describe('invariants content is suitable for system_prompt injection', () => {
|
|
31
|
+
it('non-empty content can be concatenated into a system_prompt string', async () => {
|
|
32
|
+
const dir = join('/tmp', '.studio-invariants-integration-' + Date.now());
|
|
33
|
+
await mkdir(dir, { recursive: true });
|
|
34
|
+
await writeFile(
|
|
35
|
+
join(dir, 'invariants.md'),
|
|
36
|
+
'## Invariants\n\n- Never hallucinate entity names\n- Cite sources'
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
const content = await loadInvariantsFile(dir);
|
|
40
|
+
|
|
41
|
+
expect(content).toBeDefined();
|
|
42
|
+
expect(typeof content).toBe('string');
|
|
43
|
+
expect(content!.length).toBeGreaterThan(0);
|
|
44
|
+
|
|
45
|
+
const systemPrompt = `You are an agent.\n\n---\n\n## Project Invariants\n\n${content}`;
|
|
46
|
+
expect(systemPrompt).toContain('Never hallucinate entity names');
|
|
47
|
+
expect(systemPrompt).toContain('## Project Invariants');
|
|
48
|
+
|
|
49
|
+
await rm(dir, { recursive: true, force: true });
|
|
50
|
+
});
|
|
51
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Load `.studio/invariants.md` from the project directory.
|
|
6
|
+
* Returns content if the file exists, undefined otherwise (non-fatal).
|
|
7
|
+
*/
|
|
8
|
+
export async function loadInvariantsFile(projectDir: string): Promise<string | undefined> {
|
|
9
|
+
try {
|
|
10
|
+
return await readFile(join(projectDir, 'invariants.md'), 'utf-8');
|
|
11
|
+
} catch {
|
|
12
|
+
return undefined;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { parsePipelineYaml } from './loader.js';
|
|
3
|
+
import type { StageDefinition } from '@studio-foundation/contracts';
|
|
4
|
+
|
|
5
|
+
const MINIMAL_STAGE = `
|
|
6
|
+
- name: analyze
|
|
7
|
+
kind: analysis
|
|
8
|
+
agent: analyst
|
|
9
|
+
`;
|
|
10
|
+
|
|
11
|
+
describe('parsePipelineYaml — on_pipeline_start', () => {
|
|
12
|
+
it('parses on_pipeline_start commands', () => {
|
|
13
|
+
const yaml = `
|
|
14
|
+
name: test-pipeline
|
|
15
|
+
description: test
|
|
16
|
+
version: 1
|
|
17
|
+
on_pipeline_start:
|
|
18
|
+
- command: "git status --short"
|
|
19
|
+
inject_as: git_status
|
|
20
|
+
- command: "git log --oneline -5"
|
|
21
|
+
inject_as: recent_commits
|
|
22
|
+
stages:
|
|
23
|
+
${MINIMAL_STAGE}
|
|
24
|
+
`;
|
|
25
|
+
const result = parsePipelineYaml(yaml);
|
|
26
|
+
expect(result.on_pipeline_start).toEqual([
|
|
27
|
+
{ command: 'git status --short', inject_as: 'git_status' },
|
|
28
|
+
{ command: 'git log --oneline -5', inject_as: 'recent_commits' },
|
|
29
|
+
]);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it('returns undefined on_pipeline_start when absent', () => {
|
|
33
|
+
const yaml = `
|
|
34
|
+
name: test-pipeline
|
|
35
|
+
description: test
|
|
36
|
+
version: 1
|
|
37
|
+
stages:
|
|
38
|
+
${MINIMAL_STAGE}
|
|
39
|
+
`;
|
|
40
|
+
const result = parsePipelineYaml(yaml);
|
|
41
|
+
expect(result.on_pipeline_start).toBeUndefined();
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('throws when on_pipeline_start entry is missing command', () => {
|
|
45
|
+
const yaml = `
|
|
46
|
+
name: test-pipeline
|
|
47
|
+
description: test
|
|
48
|
+
version: 1
|
|
49
|
+
on_pipeline_start:
|
|
50
|
+
- inject_as: git_status
|
|
51
|
+
stages:
|
|
52
|
+
${MINIMAL_STAGE}
|
|
53
|
+
`;
|
|
54
|
+
expect(() => parsePipelineYaml(yaml)).toThrow("on_pipeline_start entry missing 'command'");
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('throws when on_pipeline_start entry is missing inject_as', () => {
|
|
58
|
+
const yaml = `
|
|
59
|
+
name: test-pipeline
|
|
60
|
+
description: test
|
|
61
|
+
version: 1
|
|
62
|
+
on_pipeline_start:
|
|
63
|
+
- command: "git status"
|
|
64
|
+
stages:
|
|
65
|
+
${MINIMAL_STAGE}
|
|
66
|
+
`;
|
|
67
|
+
expect(() => parsePipelineYaml(yaml)).toThrow("on_pipeline_start entry missing 'inject_as'");
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
const PIPELINE_WITH_HOOKS = `
|
|
72
|
+
name: test-pipeline
|
|
73
|
+
description: test
|
|
74
|
+
version: 1
|
|
75
|
+
stages:
|
|
76
|
+
- name: code-gen
|
|
77
|
+
kind: code
|
|
78
|
+
agent: coder
|
|
79
|
+
hooks:
|
|
80
|
+
on_stage_start:
|
|
81
|
+
- command: "git stash"
|
|
82
|
+
on_failure: warn
|
|
83
|
+
on_stage_complete:
|
|
84
|
+
- command: "npx tsc --noEmit"
|
|
85
|
+
on_failure: reject
|
|
86
|
+
pre_tool_use:
|
|
87
|
+
- matcher: "repo_manager-write_file"
|
|
88
|
+
command: "echo pre {{tool.path}}"
|
|
89
|
+
on_failure: warn
|
|
90
|
+
post_tool_use:
|
|
91
|
+
- matcher: "repo_manager-write_file"
|
|
92
|
+
command: "npx prettier --write {{tool.path}}"
|
|
93
|
+
on_failure: warn
|
|
94
|
+
`;
|
|
95
|
+
|
|
96
|
+
describe('parsePipelineYaml — stage hooks', () => {
|
|
97
|
+
it('parses all four hook types from a stage', () => {
|
|
98
|
+
const result = parsePipelineYaml(PIPELINE_WITH_HOOKS);
|
|
99
|
+
const stage = result.stages[0] as StageDefinition;
|
|
100
|
+
expect(stage.hooks?.on_stage_start).toEqual([
|
|
101
|
+
{ command: 'git stash', on_failure: 'warn' },
|
|
102
|
+
]);
|
|
103
|
+
expect(stage.hooks?.on_stage_complete).toEqual([
|
|
104
|
+
{ command: 'npx tsc --noEmit', on_failure: 'reject' },
|
|
105
|
+
]);
|
|
106
|
+
expect(stage.hooks?.pre_tool_use).toEqual([
|
|
107
|
+
{ matcher: 'repo_manager-write_file', command: 'echo pre {{tool.path}}', on_failure: 'warn' },
|
|
108
|
+
]);
|
|
109
|
+
expect(stage.hooks?.post_tool_use).toEqual([
|
|
110
|
+
{ matcher: 'repo_manager-write_file', command: 'npx prettier --write {{tool.path}}', on_failure: 'warn' },
|
|
111
|
+
]);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('returns undefined hooks when stage has no hooks', () => {
|
|
115
|
+
const yaml = `
|
|
116
|
+
name: test-pipeline
|
|
117
|
+
description: test
|
|
118
|
+
version: 1
|
|
119
|
+
stages:
|
|
120
|
+
- name: analyze
|
|
121
|
+
kind: analysis
|
|
122
|
+
agent: analyst
|
|
123
|
+
`;
|
|
124
|
+
const result = parsePipelineYaml(yaml);
|
|
125
|
+
const stage = result.stages[0] as StageDefinition;
|
|
126
|
+
expect(stage.hooks).toBeUndefined();
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
// Load pipeline definitions from YAML files
|
|
2
|
+
|
|
3
|
+
import { readFile } from 'node:fs/promises';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import * as yaml from 'js-yaml';
|
|
6
|
+
import type { PipelineDefinition, PipelineEntry, StageGroup, StageDefinition, StartupCommand, StageHooks } from '@studio-foundation/contracts';
|
|
7
|
+
|
|
8
|
+
export async function loadPipeline(path: string): Promise<PipelineDefinition> {
|
|
9
|
+
let content: string;
|
|
10
|
+
try {
|
|
11
|
+
content = await readFile(path, 'utf-8');
|
|
12
|
+
} catch (err) {
|
|
13
|
+
throw new Error(`Failed to load pipeline at ${path}: ${(err as Error).message}`);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
return parsePipelineYaml(content, path);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function loadPipelineByName(
|
|
20
|
+
name: string,
|
|
21
|
+
pipelinesDir: string
|
|
22
|
+
): Promise<PipelineDefinition> {
|
|
23
|
+
const path = join(pipelinesDir, `${name}.pipeline.yaml`);
|
|
24
|
+
return loadPipeline(path);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function parsePipelineYaml(yamlContent: string, sourcePath?: string): PipelineDefinition {
|
|
28
|
+
const parsed = yaml.load(yamlContent) as Record<string, unknown>;
|
|
29
|
+
const context = sourcePath ? ` (from ${sourcePath})` : '';
|
|
30
|
+
|
|
31
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
32
|
+
throw new Error(`Invalid pipeline YAML: expected an object${context}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (!parsed.name || typeof parsed.name !== 'string') {
|
|
36
|
+
throw new Error(`Pipeline missing required field 'name'${context}`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (!Array.isArray(parsed.stages)) {
|
|
40
|
+
throw new Error(`Pipeline missing required field 'stages'${context}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (parsed.stages.length === 0) {
|
|
44
|
+
throw new Error(`Pipeline must have at least one stage${context}`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const stages: PipelineEntry[] = [];
|
|
48
|
+
for (const entry of parsed.stages as any[]) {
|
|
49
|
+
if (entry.group) {
|
|
50
|
+
// Group entry
|
|
51
|
+
if (!Array.isArray(entry.stages) || entry.stages.length < 2) {
|
|
52
|
+
throw new Error(`Group '${entry.group}' must have at least 2 stages${context}`);
|
|
53
|
+
}
|
|
54
|
+
for (const s of entry.stages) {
|
|
55
|
+
validateStageFields(s, context);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const mode = entry.mode === 'parallel' ? 'parallel' : undefined;
|
|
59
|
+
let maxIterations: number = entry.max_iterations ?? 3;
|
|
60
|
+
|
|
61
|
+
if (mode === 'parallel' && maxIterations > 1) {
|
|
62
|
+
console.warn(
|
|
63
|
+
`[studio] parallel group '${entry.group}' has max_iterations > 1 — iterations are ignored in parallel mode, using 1`
|
|
64
|
+
);
|
|
65
|
+
maxIterations = 1;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
stages.push({
|
|
69
|
+
group: entry.group,
|
|
70
|
+
max_iterations: maxIterations,
|
|
71
|
+
...(mode ? { mode } : {}),
|
|
72
|
+
...(entry.on_failure ? { on_failure: entry.on_failure } : {}),
|
|
73
|
+
stages: entry.stages.map((s: any) => ({ ...s, hooks: parseStageHooks(s) })),
|
|
74
|
+
} as StageGroup);
|
|
75
|
+
} else {
|
|
76
|
+
// Simple stage
|
|
77
|
+
validateStageFields(entry, context);
|
|
78
|
+
stages.push({ ...entry, hooks: parseStageHooks(entry) } as StageDefinition);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Parse on_pipeline_start commands
|
|
83
|
+
let on_pipeline_start: StartupCommand[] | undefined;
|
|
84
|
+
if (Array.isArray(parsed.on_pipeline_start)) {
|
|
85
|
+
on_pipeline_start = [];
|
|
86
|
+
for (const cmd of parsed.on_pipeline_start as any[]) {
|
|
87
|
+
if (!cmd.command || typeof cmd.command !== 'string') {
|
|
88
|
+
throw new Error(`on_pipeline_start entry missing 'command'${context}`);
|
|
89
|
+
}
|
|
90
|
+
if (!cmd.inject_as || typeof cmd.inject_as !== 'string') {
|
|
91
|
+
throw new Error(`on_pipeline_start entry missing 'inject_as'${context}`);
|
|
92
|
+
}
|
|
93
|
+
on_pipeline_start.push({ command: cmd.command, inject_as: cmd.inject_as });
|
|
94
|
+
}
|
|
95
|
+
if (on_pipeline_start.length === 0) {
|
|
96
|
+
on_pipeline_start = undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return {
|
|
101
|
+
...parsed,
|
|
102
|
+
stages,
|
|
103
|
+
on_pipeline_start,
|
|
104
|
+
} as unknown as PipelineDefinition;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function parseStageHooks(entry: any): StageHooks | undefined {
|
|
108
|
+
if (!entry.hooks || typeof entry.hooks !== 'object') return undefined;
|
|
109
|
+
const h = entry.hooks;
|
|
110
|
+
const result: StageHooks = {};
|
|
111
|
+
|
|
112
|
+
if (Array.isArray(h.on_stage_start)) {
|
|
113
|
+
result.on_stage_start = h.on_stage_start.map((hk: any) => ({
|
|
114
|
+
command: String(hk.command ?? ''),
|
|
115
|
+
on_failure: hk.on_failure ?? 'warn',
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
if (Array.isArray(h.on_stage_complete)) {
|
|
119
|
+
result.on_stage_complete = h.on_stage_complete.map((hk: any) => ({
|
|
120
|
+
command: String(hk.command ?? ''),
|
|
121
|
+
on_failure: hk.on_failure ?? 'warn',
|
|
122
|
+
}));
|
|
123
|
+
}
|
|
124
|
+
if (Array.isArray(h.pre_tool_use)) {
|
|
125
|
+
result.pre_tool_use = h.pre_tool_use.map((hk: any) => ({
|
|
126
|
+
matcher: String(hk.matcher ?? ''),
|
|
127
|
+
command: String(hk.command ?? ''),
|
|
128
|
+
on_failure: hk.on_failure ?? 'warn',
|
|
129
|
+
}));
|
|
130
|
+
}
|
|
131
|
+
if (Array.isArray(h.post_tool_use)) {
|
|
132
|
+
result.post_tool_use = h.post_tool_use.map((hk: any) => ({
|
|
133
|
+
matcher: String(hk.matcher ?? ''),
|
|
134
|
+
command: String(hk.command ?? ''),
|
|
135
|
+
on_failure: hk.on_failure ?? 'warn',
|
|
136
|
+
}));
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const hasAny = result.on_stage_start || result.on_stage_complete
|
|
140
|
+
|| result.pre_tool_use || result.post_tool_use;
|
|
141
|
+
return hasAny ? result : undefined;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function validateStageFields(stage: any, context: string): void {
|
|
145
|
+
if (!stage.name) throw new Error(`Stage missing 'name'${context}`);
|
|
146
|
+
if (!stage.kind && !stage.executor) throw new Error(`Stage '${stage.name}' missing 'kind'${context}`);
|
|
147
|
+
if (!stage.agent && stage.executor !== 'script') throw new Error(`Stage '${stage.name}' missing 'agent'${context}`);
|
|
148
|
+
if (stage.executor === 'script' && !stage.script) throw new Error(`Stage '${stage.name}' missing 'script' (required when executor: 'script')${context}`);
|
|
149
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { validateOutput } from './output-validator.js';
|
|
3
|
+
import type { OutputContract } from '@studio-foundation/contracts';
|
|
4
|
+
|
|
5
|
+
const schemaOnlyContract: OutputContract = {
|
|
6
|
+
name: 'test-schema',
|
|
7
|
+
version: 1,
|
|
8
|
+
schema: { required_fields: ['summary', 'files_changed'] },
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const toolCallContract: OutputContract = {
|
|
12
|
+
name: 'test-tool-calls',
|
|
13
|
+
version: 1,
|
|
14
|
+
schema: { required_fields: ['summary'] },
|
|
15
|
+
tool_calls: { minimum: 1, required_tools: ['repo_manager-write_file'] },
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const postValidationContract: OutputContract = {
|
|
19
|
+
name: 'test-post-validation',
|
|
20
|
+
version: 1,
|
|
21
|
+
schema: { required_fields: ['status'] },
|
|
22
|
+
post_validation: {
|
|
23
|
+
rejection_detection: {
|
|
24
|
+
field: 'status',
|
|
25
|
+
approved_values: ['approved'],
|
|
26
|
+
rejected_values: ['rejected'],
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
describe('validateOutput', () => {
|
|
32
|
+
describe('schema validation', () => {
|
|
33
|
+
it('returns valid: true when all required fields are present', () => {
|
|
34
|
+
const result = validateOutput(schemaOnlyContract, { summary: 'ok', files_changed: ['a.ts'] });
|
|
35
|
+
expect(result.valid).toBe(true);
|
|
36
|
+
expect(result.errors).toEqual([]);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('returns valid: false with error when required field is missing', () => {
|
|
40
|
+
const result = validateOutput(schemaOnlyContract, { summary: 'ok' });
|
|
41
|
+
expect(result.valid).toBe(false);
|
|
42
|
+
expect(result.errors).toContain('Missing required field: files_changed');
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('returns valid: false when output is not an object', () => {
|
|
46
|
+
const result = validateOutput(schemaOnlyContract, 'not an object');
|
|
47
|
+
expect(result.valid).toBe(false);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe('tool_calls validation', () => {
|
|
52
|
+
it('returns valid: false when minimum not met (empty tool_calls)', () => {
|
|
53
|
+
const result = validateOutput(toolCallContract, { summary: 'ok' }, []);
|
|
54
|
+
expect(result.valid).toBe(false);
|
|
55
|
+
expect(result.errors.some(e => e.includes('at least 1 successful tool call'))).toBe(true);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('returns valid: false when required tool was not called', () => {
|
|
59
|
+
const result = validateOutput(
|
|
60
|
+
toolCallContract,
|
|
61
|
+
{ summary: 'ok' },
|
|
62
|
+
[{ id: 'call-1', name: 'shell-run_command', arguments: {}, result: 'ok' }]
|
|
63
|
+
);
|
|
64
|
+
expect(result.valid).toBe(false);
|
|
65
|
+
expect(result.errors.some(e => e.includes('repo_manager-write_file'))).toBe(true);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it('returns valid: true when required tool was called successfully', () => {
|
|
69
|
+
const result = validateOutput(
|
|
70
|
+
toolCallContract,
|
|
71
|
+
{ summary: 'ok' },
|
|
72
|
+
[{ id: 'call-2', name: 'repo_manager-write_file', arguments: { path: 'a.ts' }, result: 'ok' }]
|
|
73
|
+
);
|
|
74
|
+
expect(result.valid).toBe(true);
|
|
75
|
+
expect(result.errors).toEqual([]);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
describe('post_validation', () => {
|
|
80
|
+
it('accepted: true when approved value present', () => {
|
|
81
|
+
const result = validateOutput(postValidationContract, { status: 'approved' });
|
|
82
|
+
expect(result.valid).toBe(true);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('accepted: false with rejection_reason when rejected value present', () => {
|
|
86
|
+
const result = validateOutput(postValidationContract, { status: 'rejected' });
|
|
87
|
+
expect(result.post_validation.accepted).toBe(false);
|
|
88
|
+
expect(result.post_validation.rejection_reason).toBeTruthy();
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('post_validation runs independently of schema validity', () => {
|
|
92
|
+
const result = validateOutput(
|
|
93
|
+
{ ...postValidationContract, schema: { required_fields: ['summary', 'status'] } },
|
|
94
|
+
{ status: 'rejected' }
|
|
95
|
+
);
|
|
96
|
+
expect(result.valid).toBe(false);
|
|
97
|
+
expect(result.post_validation.accepted).toBe(false);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
});
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { OutputContract, ToolCall } from '@studio-foundation/contracts';
|
|
2
|
+
import {
|
|
3
|
+
validateSchema,
|
|
4
|
+
validateToolCalls,
|
|
5
|
+
validateRequiredTools,
|
|
6
|
+
validateCountedTools,
|
|
7
|
+
validateToolGroups,
|
|
8
|
+
} from '@studio-foundation/ralph';
|
|
9
|
+
import { postValidate, type PostValidationResult } from './post-validator.js';
|
|
10
|
+
|
|
11
|
+
export interface OutputValidationResult {
|
|
12
|
+
valid: boolean;
|
|
13
|
+
errors: string[];
|
|
14
|
+
warnings: string[];
|
|
15
|
+
post_validation: PostValidationResult;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function validateOutput(
|
|
19
|
+
contract: OutputContract,
|
|
20
|
+
output: unknown,
|
|
21
|
+
toolCalls: ToolCall[] = []
|
|
22
|
+
): OutputValidationResult {
|
|
23
|
+
const results = [
|
|
24
|
+
validateSchema(output, contract),
|
|
25
|
+
validateToolCalls(toolCalls, contract.tool_calls),
|
|
26
|
+
validateRequiredTools(toolCalls, contract.tool_calls),
|
|
27
|
+
validateCountedTools(toolCalls, contract.tool_calls),
|
|
28
|
+
validateToolGroups(toolCalls, contract.tool_calls),
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
const errors = results.flatMap(r => r.errors);
|
|
32
|
+
const warnings = results.flatMap(r => r.warnings);
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
valid: errors.length === 0,
|
|
36
|
+
errors,
|
|
37
|
+
warnings,
|
|
38
|
+
post_validation: postValidate(output, contract),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Post-validation sémantique
|
|
2
|
+
//
|
|
3
|
+
// Vérifie le CONTENU de l'output après que ralph a validé le FORMAT.
|
|
4
|
+
// Utilisé pour les stages avec une gate d'approbation — l'agent peut
|
|
5
|
+
// retourner un JSON valide qui dit quand même "non".
|
|
6
|
+
//
|
|
7
|
+
// Configuré dans le contract YAML via la section "post_validation".
|
|
8
|
+
|
|
9
|
+
import type { OutputContract } from '@studio-foundation/contracts';
|
|
10
|
+
|
|
11
|
+
export interface PostValidationResult {
|
|
12
|
+
accepted: boolean;
|
|
13
|
+
rejection_reason?: string;
|
|
14
|
+
rejection_details?: string[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function postValidate(
|
|
18
|
+
output: unknown,
|
|
19
|
+
contract: OutputContract
|
|
20
|
+
): PostValidationResult {
|
|
21
|
+
// No post_validation config → everything is accepted
|
|
22
|
+
if (!contract.post_validation?.rejection_detection) {
|
|
23
|
+
return { accepted: true };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const { field, approved_values, rejected_values, details_field, summary_field, reject_if_non_empty } =
|
|
27
|
+
contract.post_validation.rejection_detection;
|
|
28
|
+
|
|
29
|
+
if (!field) {
|
|
30
|
+
return { accepted: true };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Extract field value from output
|
|
34
|
+
if (!output || typeof output !== 'object') {
|
|
35
|
+
return { accepted: true };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const o = output as Record<string, unknown>;
|
|
39
|
+
const actualValue = o[field];
|
|
40
|
+
|
|
41
|
+
if (typeof actualValue !== 'string') {
|
|
42
|
+
return { accepted: true };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Check reject_if_non_empty before approved_values — a non-empty field always rejects
|
|
46
|
+
if (reject_if_non_empty) {
|
|
47
|
+
const fieldValue = o[reject_if_non_empty];
|
|
48
|
+
if (Array.isArray(fieldValue) && fieldValue.length > 0) {
|
|
49
|
+
const details = fieldValue
|
|
50
|
+
.map(item => {
|
|
51
|
+
if (typeof item === 'string') return item;
|
|
52
|
+
if (typeof item !== 'object' || item === null) return undefined;
|
|
53
|
+
const obj = item as Record<string, unknown>;
|
|
54
|
+
const text = obj.description ?? obj.issue ?? obj.message ?? obj.text ?? obj.error;
|
|
55
|
+
if (typeof text === 'string' && text.length > 0) return text;
|
|
56
|
+
return Object.values(obj).find((v): v is string => typeof v === 'string' && v.length > 0);
|
|
57
|
+
})
|
|
58
|
+
.filter((d): d is string => typeof d === 'string');
|
|
59
|
+
|
|
60
|
+
const summary = summary_field && typeof o[summary_field] === 'string'
|
|
61
|
+
? (o[summary_field] as string)
|
|
62
|
+
: undefined;
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
accepted: false,
|
|
66
|
+
rejection_reason: `Rejected: ${reject_if_non_empty} is non-empty (${fieldValue.length} items)${summary ? `. ${summary}` : ''}`,
|
|
67
|
+
rejection_details: details.length > 0 ? details : undefined,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Check approved values (if specified)
|
|
73
|
+
if (approved_values?.length && approved_values.includes(actualValue)) {
|
|
74
|
+
return { accepted: true };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Check rejected values (if specified)
|
|
78
|
+
if (rejected_values?.length && !rejected_values.includes(actualValue)) {
|
|
79
|
+
// Value is not in rejected list and no approved list matched → accept
|
|
80
|
+
if (!approved_values?.length) {
|
|
81
|
+
return { accepted: true };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// If we have approved_values and the value isn't in them → rejected
|
|
86
|
+
// If we have rejected_values and the value is in them → rejected
|
|
87
|
+
|
|
88
|
+
// Extract details from configured field
|
|
89
|
+
const details: string[] = [];
|
|
90
|
+
if (details_field) {
|
|
91
|
+
const detailsValue = o[details_field];
|
|
92
|
+
if (typeof detailsValue === 'string' && detailsValue.length > 0) {
|
|
93
|
+
details.push(detailsValue);
|
|
94
|
+
} else if (Array.isArray(detailsValue)) {
|
|
95
|
+
for (const item of detailsValue) {
|
|
96
|
+
if (typeof item === 'string') {
|
|
97
|
+
details.push(item);
|
|
98
|
+
} else if (typeof item === 'object' && item !== null) {
|
|
99
|
+
const obj = item as Record<string, unknown>;
|
|
100
|
+
const text = obj.description ?? obj.issue ?? obj.message ?? obj.text ?? obj.error;
|
|
101
|
+
if (typeof text === 'string' && text.length > 0) {
|
|
102
|
+
details.push(text);
|
|
103
|
+
} else {
|
|
104
|
+
const firstStr = Object.values(obj).find(
|
|
105
|
+
(v): v is string => typeof v === 'string' && v.length > 0
|
|
106
|
+
);
|
|
107
|
+
if (firstStr !== undefined) details.push(firstStr);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Extract summary from configured field
|
|
115
|
+
const summary = summary_field && typeof o[summary_field] === 'string'
|
|
116
|
+
? (o[summary_field] as string)
|
|
117
|
+
: undefined;
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
accepted: false,
|
|
121
|
+
rejection_reason: `Rejected: ${field} = "${actualValue}" (expected: ${(approved_values ?? []).join(' or ')})${summary ? `. ${summary}` : ''}`,
|
|
122
|
+
rejection_details: details.length > 0 ? details : undefined,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest';
|
|
2
|
+
import { mkdir, writeFile, rm } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { loadSkillFiles } from './skill-loader.js';
|
|
5
|
+
|
|
6
|
+
const TMP = join('/tmp', '.studio-skill-loader-test-' + Date.now());
|
|
7
|
+
|
|
8
|
+
describe('loadSkillFiles', () => {
|
|
9
|
+
beforeAll(async () => {
|
|
10
|
+
await mkdir(TMP, { recursive: true });
|
|
11
|
+
await writeFile(join(TMP, 'git-workflow.skill.md'), '# Git Workflow\n\nAlways branch from main.');
|
|
12
|
+
await writeFile(join(TMP, 'code-conventions.skill.md'), '# Code Conventions\n\nUse camelCase.');
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
afterAll(async () => {
|
|
16
|
+
await rm(TMP, { recursive: true, force: true });
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('loads existing skill files by name', async () => {
|
|
20
|
+
const skills = await loadSkillFiles(['git-workflow', 'code-conventions'], TMP);
|
|
21
|
+
expect(skills).toHaveLength(2);
|
|
22
|
+
expect(skills[0]).toEqual({ name: 'git-workflow', content: '# Git Workflow\n\nAlways branch from main.' });
|
|
23
|
+
expect(skills[1]).toEqual({ name: 'code-conventions', content: '# Code Conventions\n\nUse camelCase.' });
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('skips missing skill files without throwing', async () => {
|
|
27
|
+
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
28
|
+
const skills = await loadSkillFiles(['git-workflow', 'nonexistent'], TMP);
|
|
29
|
+
expect(skills).toHaveLength(1);
|
|
30
|
+
expect(skills[0].name).toBe('git-workflow');
|
|
31
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('nonexistent'));
|
|
32
|
+
warnSpy.mockRestore();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('returns empty array when names list is empty', async () => {
|
|
36
|
+
const skills = await loadSkillFiles([], TMP);
|
|
37
|
+
expect(skills).toHaveLength(0);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('returns empty array when skills directory does not exist', async () => {
|
|
41
|
+
const skills = await loadSkillFiles(['git-workflow'], '/tmp/nonexistent-skills-dir-xyz-abc');
|
|
42
|
+
expect(skills).toHaveLength(0);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
|
|
5
|
+
export interface SkillContent {
|
|
6
|
+
name: string;
|
|
7
|
+
content: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Load `.skill.md` files by name from a skills directory.
|
|
12
|
+
* Missing files are skipped with a warning (non-fatal).
|
|
13
|
+
*/
|
|
14
|
+
export async function loadSkillFiles(
|
|
15
|
+
names: string[],
|
|
16
|
+
skillsDir: string
|
|
17
|
+
): Promise<SkillContent[]> {
|
|
18
|
+
if (names.length === 0) return [];
|
|
19
|
+
if (!existsSync(skillsDir)) return [];
|
|
20
|
+
|
|
21
|
+
const results: SkillContent[] = [];
|
|
22
|
+
for (const name of names) {
|
|
23
|
+
const filePath = join(skillsDir, `${name}.skill.md`);
|
|
24
|
+
try {
|
|
25
|
+
const content = await readFile(filePath, 'utf-8');
|
|
26
|
+
results.push({ name, content });
|
|
27
|
+
} catch {
|
|
28
|
+
console.warn(`[studio] Skill '${name}' not found at ${filePath} — skipping.`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return results;
|
|
32
|
+
}
|