@interf/compiler 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -66
- package/builtin-workflows/interf/README.md +6 -6
- package/builtin-workflows/interf/compile/stages/shape/SKILL.md +7 -7
- package/builtin-workflows/interf/compile/stages/structure/SKILL.md +2 -2
- package/builtin-workflows/interf/compile/stages/summarize/SKILL.md +1 -1
- package/builtin-workflows/interf/{workspace.schema.json → compiled.schema.json} +5 -5
- package/builtin-workflows/interf/improve/SKILL.md +3 -3
- package/builtin-workflows/interf/use/query/SKILL.md +2 -2
- package/builtin-workflows/interf/workflow.json +42 -31
- package/dist/commands/check-draft.d.ts +19 -0
- package/dist/commands/check-draft.js +110 -0
- package/dist/commands/compile-controller.d.ts +4 -4
- package/dist/commands/compile-controller.js +117 -81
- package/dist/commands/compile.d.ts +5 -5
- package/dist/commands/compile.js +61 -62
- package/dist/commands/compiled-flow.d.ts +23 -0
- package/dist/commands/compiled-flow.js +112 -0
- package/dist/commands/create-workflow-wizard.d.ts +3 -3
- package/dist/commands/create-workflow-wizard.js +11 -11
- package/dist/commands/create.d.ts +2 -2
- package/dist/commands/create.js +50 -57
- package/dist/commands/default.js +2 -2
- package/dist/commands/executor-flow.d.ts +20 -1
- package/dist/commands/executor-flow.js +67 -7
- package/dist/commands/init.js +242 -289
- package/dist/commands/list.js +14 -10
- package/dist/commands/reset.js +6 -6
- package/dist/commands/source-config-wizard.d.ts +12 -8
- package/dist/commands/source-config-wizard.js +356 -119
- package/dist/commands/status.js +49 -26
- package/dist/commands/test-flow.d.ts +23 -10
- package/dist/commands/test-flow.js +274 -65
- package/dist/commands/test.d.ts +7 -1
- package/dist/commands/test.js +264 -65
- package/dist/commands/verify.js +23 -14
- package/dist/index.d.ts +7 -7
- package/dist/index.js +4 -4
- package/dist/lib/agent-args.js +2 -1
- package/dist/lib/agent-constants.js +1 -1
- package/dist/lib/agent-render.js +4 -4
- package/dist/lib/agent-shells.d.ts +8 -8
- package/dist/lib/agent-shells.js +231 -142
- package/dist/lib/{workflow-abi.d.ts → builtin-compiled-workflow.d.ts} +37 -46
- package/dist/lib/builtin-compiled-workflow.js +153 -0
- package/dist/lib/compiled-compile.d.ts +52 -0
- package/dist/lib/compiled-compile.js +274 -0
- package/dist/lib/compiled-home.d.ts +5 -0
- package/dist/lib/compiled-home.js +32 -0
- package/dist/lib/compiled-paths.d.ts +39 -0
- package/dist/lib/compiled-paths.js +103 -0
- package/dist/lib/{workspace-raw.d.ts → compiled-raw.d.ts} +9 -8
- package/dist/lib/{workspace-raw.js → compiled-raw.js} +16 -14
- package/dist/lib/compiled-reset.d.ts +1 -0
- package/dist/lib/compiled-reset.js +44 -0
- package/dist/lib/compiled-schema.d.ts +27 -0
- package/dist/lib/compiled-schema.js +110 -0
- package/dist/lib/config.d.ts +0 -1
- package/dist/lib/config.js +0 -1
- package/dist/lib/discovery.d.ts +1 -1
- package/dist/lib/discovery.js +3 -3
- package/dist/lib/interf-bootstrap.d.ts +1 -1
- package/dist/lib/interf-bootstrap.js +4 -4
- package/dist/lib/interf-detect.d.ts +9 -10
- package/dist/lib/interf-detect.js +70 -59
- package/dist/lib/interf-scaffold.d.ts +2 -2
- package/dist/lib/interf-scaffold.js +90 -57
- package/dist/lib/interf-workflow-package.d.ts +3 -3
- package/dist/lib/interf-workflow-package.js +30 -30
- package/dist/lib/interf.d.ts +5 -5
- package/dist/lib/interf.js +4 -4
- package/dist/lib/local-workflows.d.ts +4 -4
- package/dist/lib/local-workflows.js +35 -70
- package/dist/lib/obsidian.d.ts +1 -1
- package/dist/lib/parse.js +92 -1
- package/dist/lib/project-paths.d.ts +11 -0
- package/dist/lib/project-paths.js +32 -0
- package/dist/lib/runtime-acceptance.d.ts +7 -1
- package/dist/lib/runtime-acceptance.js +194 -59
- package/dist/lib/runtime-contracts.d.ts +2 -4
- package/dist/lib/runtime-contracts.js +17 -161
- package/dist/lib/runtime-inventory.d.ts +7 -0
- package/dist/lib/runtime-inventory.js +29 -0
- package/dist/lib/runtime-paths.js +5 -5
- package/dist/lib/runtime-prompt.js +7 -6
- package/dist/lib/runtime-reconcile.d.ts +2 -3
- package/dist/lib/runtime-reconcile.js +94 -184
- package/dist/lib/runtime-runs.js +25 -119
- package/dist/lib/runtime-types.d.ts +10 -19
- package/dist/lib/runtime.d.ts +2 -2
- package/dist/lib/runtime.js +1 -1
- package/dist/lib/schema.d.ts +169 -153
- package/dist/lib/schema.js +116 -164
- package/dist/lib/source-config.d.ts +24 -20
- package/dist/lib/source-config.js +159 -122
- package/dist/lib/state-artifacts.d.ts +5 -5
- package/dist/lib/state-artifacts.js +8 -8
- package/dist/lib/state-health.d.ts +4 -4
- package/dist/lib/state-health.js +110 -126
- package/dist/lib/state-io.d.ts +8 -8
- package/dist/lib/state-io.js +21 -102
- package/dist/lib/state-paths.js +5 -5
- package/dist/lib/state-view.d.ts +4 -4
- package/dist/lib/state-view.js +52 -55
- package/dist/lib/state.d.ts +5 -5
- package/dist/lib/state.js +4 -4
- package/dist/lib/summarize-plan.d.ts +3 -2
- package/dist/lib/summarize-plan.js +19 -21
- package/dist/lib/test-execution.js +9 -9
- package/dist/lib/test-matrices.d.ts +3 -3
- package/dist/lib/test-matrices.js +6 -6
- package/dist/lib/test-paths.d.ts +4 -4
- package/dist/lib/test-paths.js +26 -11
- package/dist/lib/test-sandbox.d.ts +1 -1
- package/dist/lib/test-sandbox.js +32 -38
- package/dist/lib/test-specs.js +1 -1
- package/dist/lib/test-targets.d.ts +2 -2
- package/dist/lib/test-targets.js +11 -11
- package/dist/lib/test-types.d.ts +1 -1
- package/dist/lib/test.d.ts +1 -1
- package/dist/lib/test.js +1 -1
- package/dist/lib/util.d.ts +2 -0
- package/dist/lib/util.js +14 -1
- package/dist/lib/validate-compiled.d.ts +27 -0
- package/dist/lib/validate-compiled.js +238 -0
- package/dist/lib/validate-helpers.d.ts +0 -8
- package/dist/lib/validate-helpers.js +0 -30
- package/dist/lib/validate.d.ts +6 -4
- package/dist/lib/validate.js +76 -27
- package/dist/lib/workflow-definitions.d.ts +12 -11
- package/dist/lib/workflow-definitions.js +45 -55
- package/dist/lib/workflow-helpers.d.ts +2 -3
- package/dist/lib/workflow-helpers.js +9 -13
- package/dist/lib/workflow-improvement.d.ts +3 -3
- package/dist/lib/workflow-improvement.js +48 -48
- package/dist/lib/workflow-primitives.d.ts +2 -0
- package/dist/lib/workflow-primitives.js +5 -0
- package/dist/lib/workflow-review-paths.d.ts +3 -3
- package/dist/lib/workflow-review-paths.js +11 -11
- package/dist/lib/workflow-stage-runner.d.ts +1 -1
- package/dist/lib/workflow-stage-runner.js +8 -8
- package/dist/lib/workflows.d.ts +9 -9
- package/dist/lib/workflows.js +15 -17
- package/package.json +13 -12
- package/dist/commands/workspace-flow.d.ts +0 -23
- package/dist/commands/workspace-flow.js +0 -109
- package/dist/lib/registry.d.ts +0 -16
- package/dist/lib/registry.js +0 -65
- package/dist/lib/validate-workspace.d.ts +0 -121
- package/dist/lib/validate-workspace.js +0 -407
- package/dist/lib/workflow-abi.js +0 -181
- package/dist/lib/workspace-compile.d.ts +0 -54
- package/dist/lib/workspace-compile.js +0 -476
- package/dist/lib/workspace-home.d.ts +0 -5
- package/dist/lib/workspace-home.js +0 -32
- package/dist/lib/workspace-layout.d.ts +0 -2
- package/dist/lib/workspace-layout.js +0 -60
- package/dist/lib/workspace-paths.d.ts +0 -41
- package/dist/lib/workspace-paths.js +0 -107
- package/dist/lib/workspace-reset.d.ts +0 -1
- package/dist/lib/workspace-reset.js +0 -43
- package/dist/lib/workspace-schema.d.ts +0 -17
- package/dist/lib/workspace-schema.js +0 -74
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
const DraftTruthCheckSchema = z.object({
|
|
6
|
+
question: z.string().min(1),
|
|
7
|
+
answer: z.string().min(1),
|
|
8
|
+
}).strict();
|
|
9
|
+
const DraftTruthChecksSchema = z.array(DraftTruthCheckSchema).min(1).max(8);
|
|
10
|
+
export function buildTruthCheckDraftPrompt(options) {
|
|
11
|
+
const normalizedAbout = options.about?.trim();
|
|
12
|
+
const taskLines = normalizedAbout
|
|
13
|
+
? [
|
|
14
|
+
`Primary task: ${normalizedAbout}`,
|
|
15
|
+
"Draft checks that directly measure whether a local agent would be accurate on that task.",
|
|
16
|
+
"Prefer checks that would make a human confident the dataset is useful for that exact job.",
|
|
17
|
+
]
|
|
18
|
+
: [
|
|
19
|
+
"No primary task was provided.",
|
|
20
|
+
"Prefer broad, representative checks that reflect the most useful questions this dataset should answer.",
|
|
21
|
+
];
|
|
22
|
+
return [
|
|
23
|
+
"You are drafting Interf truth checks for a local dataset.",
|
|
24
|
+
"Read the files in this folder and draft a small set of checks that a human can verify from the source material.",
|
|
25
|
+
"Do not ask follow-up questions.",
|
|
26
|
+
"Keep the checks concrete, explicit, and useful for measuring agent accuracy on this dataset.",
|
|
27
|
+
"",
|
|
28
|
+
`Dataset id: ${options.datasetName}`,
|
|
29
|
+
`Dataset folder: ${options.datasetPath}`,
|
|
30
|
+
...taskLines,
|
|
31
|
+
"",
|
|
32
|
+
`Write exactly one JSON array to ${JSON.stringify(options.outputPath)}.`,
|
|
33
|
+
`Draft ${options.targetCount} checks unless the dataset is too small; in that case write fewer.`,
|
|
34
|
+
"Each array item must be an object with exactly these keys:",
|
|
35
|
+
'- "question": a plain-English question answerable from this dataset',
|
|
36
|
+
'- "answer": the expected correct answer in plain English',
|
|
37
|
+
"",
|
|
38
|
+
"Quality bar:",
|
|
39
|
+
"- If a primary task was provided, make most checks directly relevant to that task.",
|
|
40
|
+
"- Prefer facts, comparisons, or small derived calculations with clear expected answers.",
|
|
41
|
+
"- Prefer values that are easy for a human to cross-check in the source files.",
|
|
42
|
+
"- Prefer explicit chart labels, table cells, headings, totals, and directly stated comparisons.",
|
|
43
|
+
"- Avoid unstable superlatives unless the source itself clearly states the ranking in one place.",
|
|
44
|
+
"- Avoid checks where the expected answer depends on scanning many pages for a hidden exception or a larger outlier.",
|
|
45
|
+
"- Avoid vague essay questions, open-ended opinions, or anything requiring outside knowledge.",
|
|
46
|
+
"- Avoid duplicate checks that test the same fact in slightly different words.",
|
|
47
|
+
"",
|
|
48
|
+
"Output rules:",
|
|
49
|
+
"- JSON only in the output file.",
|
|
50
|
+
"- No markdown fences.",
|
|
51
|
+
"- Do not print the JSON to stdout.",
|
|
52
|
+
"- Stop after writing the file.",
|
|
53
|
+
].join("\n");
|
|
54
|
+
}
|
|
55
|
+
export async function draftTruthChecks(options) {
|
|
56
|
+
const tempDir = mkdtempSync(join(tmpdir(), "interf-draft-checks-"));
|
|
57
|
+
const outputPath = join(tempDir, "checks.json");
|
|
58
|
+
const promptPath = join(tempDir, "prompt.txt");
|
|
59
|
+
const prompt = buildTruthCheckDraftPrompt({
|
|
60
|
+
datasetName: options.datasetName,
|
|
61
|
+
datasetPath: options.datasetPath,
|
|
62
|
+
about: options.about,
|
|
63
|
+
outputPath,
|
|
64
|
+
targetCount: options.targetCount ?? 4,
|
|
65
|
+
});
|
|
66
|
+
writeFileSync(promptPath, `${prompt}\n`);
|
|
67
|
+
try {
|
|
68
|
+
const code = await options.executor.execute(options.datasetPath, prompt, {
|
|
69
|
+
completionCheck: () => existsSync(outputPath),
|
|
70
|
+
});
|
|
71
|
+
if (!existsSync(outputPath)) {
|
|
72
|
+
return {
|
|
73
|
+
checks: null,
|
|
74
|
+
error: code === 0
|
|
75
|
+
? "The local agent finished without writing draft truth checks."
|
|
76
|
+
: "The local agent did not produce draft truth checks.",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
let parsed;
|
|
80
|
+
try {
|
|
81
|
+
parsed = JSON.parse(readFileSync(outputPath, "utf8"));
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
return {
|
|
85
|
+
checks: null,
|
|
86
|
+
error: `Draft truth checks were not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
const validated = DraftTruthChecksSchema.safeParse(parsed);
|
|
90
|
+
if (!validated.success) {
|
|
91
|
+
const detail = validated.error.issues[0]?.message ?? "invalid draft truth checks";
|
|
92
|
+
return {
|
|
93
|
+
checks: null,
|
|
94
|
+
error: `Draft truth checks did not match the required shape: ${detail}`,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
checks: validated.data,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
return {
|
|
103
|
+
checks: null,
|
|
104
|
+
error: error instanceof Error ? error.message : String(error),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { WorkflowExecutionProfile, WorkflowExecutor } from "../lib/executors.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { SourceDatasetConfig } from "../lib/schema.js";
|
|
3
3
|
import type { StageShellRetentionMode } from "../lib/workflows.js";
|
|
4
|
-
export declare function
|
|
4
|
+
export declare function runConfiguredCompiledCompile(options: {
|
|
5
5
|
executor: WorkflowExecutor;
|
|
6
6
|
testExecutor?: WorkflowExecutor | null;
|
|
7
|
-
|
|
7
|
+
compiledPath: string;
|
|
8
8
|
sourcePath: string;
|
|
9
|
-
|
|
9
|
+
compiledConfig: SourceDatasetConfig | null;
|
|
10
10
|
executionProfile?: WorkflowExecutionProfile;
|
|
11
11
|
maxAttemptsOverride: number | null;
|
|
12
12
|
maxLoopsOverride: number | null;
|
|
@@ -3,43 +3,49 @@ import { tmpdir } from "node:os";
|
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
4
|
import chalk from "chalk";
|
|
5
5
|
import { readInterfConfig } from "../lib/interf.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
6
|
+
import { resetCompiledGeneratedState } from "../lib/compiled-reset.js";
|
|
7
|
+
import { formatActiveCompiledWorkflowStageStep, resolveRequiredCompiledWorkflowFromConfig, } from "../lib/workflow-definitions.js";
|
|
8
|
+
import { resolveDatasetCompileMaxAttempts, resolveDatasetCompileMaxLoops, } from "../lib/source-config.js";
|
|
9
9
|
import { runWorkflowImprovementLoop } from "../lib/workflow-improvement.js";
|
|
10
|
-
import { stageExecutionShellsRoot,
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const
|
|
10
|
+
import { stageExecutionShellsRoot, workflowPackagePathForCompiled, } from "../lib/compiled-paths.js";
|
|
11
|
+
import { readCompiledSchemaFile } from "../lib/compiled-schema.js";
|
|
12
|
+
import { compileCompiledWithReporter } from "./compiled-flow.js";
|
|
13
|
+
import { printSavedTestOutcome, questionPassRate, runSavedCompiledTest, } from "./test-flow.js";
|
|
14
|
+
function printCompileFailure(compiledPath, failedStage) {
|
|
15
|
+
const workflowId = resolveRequiredCompiledWorkflowFromConfig(readInterfConfig(compiledPath), `.interf/interf.json for ${compiledPath}`);
|
|
16
|
+
const failedStageLabel = formatActiveCompiledWorkflowStageStep(compiledPath, workflowId, failedStage ?? "compile");
|
|
16
17
|
console.log(chalk.red(` ${failedStageLabel} failed.`));
|
|
17
18
|
}
|
|
18
19
|
function testScore(outcome) {
|
|
19
20
|
return (outcome.result.passedCases * 1000) + outcome.result.passedChecks;
|
|
20
21
|
}
|
|
21
|
-
const
|
|
22
|
-
"AGENTS.md",
|
|
23
|
-
"CLAUDE.md",
|
|
24
|
-
"home.md",
|
|
25
|
-
"summaries",
|
|
26
|
-
"knowledge",
|
|
27
|
-
".claude",
|
|
28
|
-
".codex",
|
|
29
|
-
".agents",
|
|
30
|
-
".cursor",
|
|
31
|
-
".obsidian",
|
|
32
|
-
".interf/interf.json",
|
|
33
|
-
".interf/workflow",
|
|
22
|
+
const BEST_VARIATION_RUNTIME_RESTORE_PATHS = [
|
|
34
23
|
".interf/runtime/state.json",
|
|
35
24
|
".interf/runtime/health.json",
|
|
36
25
|
".interf/runtime/view-spec.json",
|
|
37
26
|
".interf/runtime/inventory.json",
|
|
38
27
|
".interf/runtime/run.json",
|
|
39
28
|
".interf/runtime/raw-snapshot.json",
|
|
40
|
-
".interf/runtime/stage-contract.json",
|
|
41
|
-
".interf/runtime/summarize-targets.json",
|
|
42
29
|
];
|
|
30
|
+
function bestVariationRestorePaths(compiledPath) {
|
|
31
|
+
const schema = readCompiledSchemaFile(workflowPackagePathForCompiled(compiledPath));
|
|
32
|
+
const zonePaths = (schema?.zones ?? [])
|
|
33
|
+
.filter((zone) => zone.id !== "raw" && zone.kind !== "runtime")
|
|
34
|
+
.map((zone) => zone.path);
|
|
35
|
+
return [
|
|
36
|
+
"AGENTS.md",
|
|
37
|
+
"CLAUDE.md",
|
|
38
|
+
".claude",
|
|
39
|
+
".codex",
|
|
40
|
+
".agents",
|
|
41
|
+
".cursor",
|
|
42
|
+
".obsidian",
|
|
43
|
+
".interf/interf.json",
|
|
44
|
+
".interf/workflow",
|
|
45
|
+
...BEST_VARIATION_RUNTIME_RESTORE_PATHS,
|
|
46
|
+
...zonePaths,
|
|
47
|
+
];
|
|
48
|
+
}
|
|
43
49
|
function copyRelativePathIfPresent(sourceRoot, targetRoot, relativePath) {
|
|
44
50
|
const sourcePath = join(sourceRoot, relativePath);
|
|
45
51
|
if (!existsSync(sourcePath))
|
|
@@ -52,18 +58,18 @@ function copyRelativePathIfPresent(sourceRoot, targetRoot, relativePath) {
|
|
|
52
58
|
errorOnExist: false,
|
|
53
59
|
});
|
|
54
60
|
}
|
|
55
|
-
function
|
|
61
|
+
function snapshotCompiledBestState(compiledPath) {
|
|
56
62
|
const snapshotRoot = mkdtempSync(join(tmpdir(), "interf-compile-attempt-"));
|
|
57
|
-
const snapshotPath = join(snapshotRoot, "
|
|
58
|
-
for (const relativePath of
|
|
59
|
-
copyRelativePathIfPresent(
|
|
63
|
+
const snapshotPath = join(snapshotRoot, "compiled");
|
|
64
|
+
for (const relativePath of bestVariationRestorePaths(compiledPath)) {
|
|
65
|
+
copyRelativePathIfPresent(compiledPath, snapshotPath, relativePath);
|
|
60
66
|
}
|
|
61
67
|
return snapshotPath;
|
|
62
68
|
}
|
|
63
|
-
function
|
|
64
|
-
for (const relativePath of
|
|
65
|
-
rmSync(join(
|
|
66
|
-
copyRelativePathIfPresent(snapshotPath,
|
|
69
|
+
function restoreCompiledBestState(snapshotPath, compiledPath) {
|
|
70
|
+
for (const relativePath of bestVariationRestorePaths(compiledPath)) {
|
|
71
|
+
rmSync(join(compiledPath, relativePath), { recursive: true, force: true });
|
|
72
|
+
copyRelativePathIfPresent(snapshotPath, compiledPath, relativePath);
|
|
67
73
|
}
|
|
68
74
|
}
|
|
69
75
|
async function runWorkflowVariation(options) {
|
|
@@ -79,21 +85,21 @@ async function runWorkflowVariation(options) {
|
|
|
79
85
|
const resetTarget = attempt > 1
|
|
80
86
|
? `attempt ${attempt}/${options.maxAttempts}`
|
|
81
87
|
: options.variationLabel;
|
|
82
|
-
console.log(chalk.dim(` Resetting generated
|
|
83
|
-
|
|
88
|
+
console.log(chalk.dim(` Resetting generated compiled state before ${resetTarget}...`));
|
|
89
|
+
resetCompiledGeneratedState(options.compiledPath, "compile");
|
|
84
90
|
}
|
|
85
|
-
const result = await
|
|
91
|
+
const result = await compileCompiledWithReporter(options.executor, options.compiledPath, {
|
|
86
92
|
heading: options.maxAttempts > 1
|
|
87
|
-
? `Building compiled
|
|
88
|
-
: `Building compiled
|
|
93
|
+
? `Building compiled dataset (${options.variationLabel}, attempt ${attempt}/${options.maxAttempts})...`
|
|
94
|
+
: `Building compiled dataset (${options.variationLabel})...`,
|
|
89
95
|
successMessage: options.maxAttempts > 1
|
|
90
|
-
? `Compiled
|
|
91
|
-
: `Compiled
|
|
96
|
+
? `Compiled dataset ready for ${options.variationLabel}, attempt ${attempt}.`
|
|
97
|
+
: `Compiled dataset ready for ${options.variationLabel}.`,
|
|
92
98
|
preserveStageShells: options.preserveStageShells,
|
|
93
99
|
});
|
|
94
100
|
if (!result.ok) {
|
|
95
|
-
printCompileFailure(options.
|
|
96
|
-
printStageShellReviewHint(options.
|
|
101
|
+
printCompileFailure(options.compiledPath, result.failedStage);
|
|
102
|
+
printStageShellReviewHint(options.compiledPath, options.preserveStageShells, "failure");
|
|
97
103
|
if (attempt < options.maxAttempts) {
|
|
98
104
|
console.log(chalk.yellow(` ${options.variationLabel} attempt ${attempt}/${options.maxAttempts} failed. Retrying with a fresh compile.`));
|
|
99
105
|
continue;
|
|
@@ -108,12 +114,12 @@ async function runWorkflowVariation(options) {
|
|
|
108
114
|
fatalError: null,
|
|
109
115
|
};
|
|
110
116
|
}
|
|
111
|
-
const outcome = await
|
|
117
|
+
const outcome = await runSavedCompiledTest({
|
|
112
118
|
sourcePath: options.sourcePath,
|
|
113
|
-
|
|
119
|
+
datasetConfig: options.compiledConfig,
|
|
114
120
|
executor: options.testExecutor,
|
|
115
121
|
executionProfile: options.executionProfile,
|
|
116
|
-
|
|
122
|
+
compiledPath: options.compiledPath,
|
|
117
123
|
preserveSandboxes: options.preserveStageShells === "always" ? "always" : "on-failure",
|
|
118
124
|
});
|
|
119
125
|
if (!outcome) {
|
|
@@ -124,7 +130,7 @@ async function runWorkflowVariation(options) {
|
|
|
124
130
|
bestOutcome,
|
|
125
131
|
bestSnapshotPath,
|
|
126
132
|
bestAttempt,
|
|
127
|
-
fatalError: "Could not run the saved
|
|
133
|
+
fatalError: "Could not run the saved compiled-dataset test after compile.",
|
|
128
134
|
};
|
|
129
135
|
}
|
|
130
136
|
console.log();
|
|
@@ -134,13 +140,13 @@ async function runWorkflowVariation(options) {
|
|
|
134
140
|
rmSync(bestSnapshotPath, { recursive: true, force: true });
|
|
135
141
|
}
|
|
136
142
|
bestOutcome = outcome;
|
|
137
|
-
bestSnapshotPath =
|
|
143
|
+
bestSnapshotPath = snapshotCompiledBestState(options.compiledPath);
|
|
138
144
|
bestAttempt = attempt;
|
|
139
145
|
}
|
|
140
146
|
if (outcome.result.ok) {
|
|
141
147
|
console.log();
|
|
142
|
-
console.log(chalk.green(`
|
|
143
|
-
printStageShellReviewHint(options.
|
|
148
|
+
console.log(chalk.green(` Compiled passed on ${options.variationLabel}, attempt ${attempt}/${options.maxAttempts}.`));
|
|
149
|
+
printStageShellReviewHint(options.compiledPath, options.preserveStageShells, "success");
|
|
144
150
|
return {
|
|
145
151
|
passed: true,
|
|
146
152
|
attemptsRun,
|
|
@@ -203,13 +209,27 @@ function summarizeWorkflowVariation(options) {
|
|
|
203
209
|
summary,
|
|
204
210
|
};
|
|
205
211
|
}
|
|
206
|
-
function printStageShellReviewHint(
|
|
207
|
-
const reviewRoot = stageExecutionShellsRoot(
|
|
212
|
+
function printStageShellReviewHint(compiledPath, preserveStageShells, result) {
|
|
213
|
+
const reviewRoot = stageExecutionShellsRoot(compiledPath);
|
|
208
214
|
if (preserveStageShells === "always") {
|
|
209
|
-
console.log(chalk.dim(`
|
|
215
|
+
console.log(chalk.dim(` Stage shells are saved under: ${reviewRoot}`));
|
|
210
216
|
return;
|
|
211
217
|
}
|
|
212
|
-
|
|
218
|
+
if (result === "failure") {
|
|
219
|
+
console.log(chalk.dim(` Review failed stage shells under: ${reviewRoot}`));
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function printPostCompileNextStep(options) {
|
|
223
|
+
console.log(chalk.dim(` Compiled dataset: ${options.compiledPath}`));
|
|
224
|
+
if (options.checks === 0) {
|
|
225
|
+
console.log(chalk.dim(" Next: run `interf` to add truth checks, then `interf test`."));
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
if (options.testedDuringCompile) {
|
|
229
|
+
console.log(chalk.dim(" Next: inspect the compiled dataset or rerun `interf test` for a fresh comparison."));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
console.log(chalk.dim(" Next: run `interf test` to compare the raw files and the compiled dataset."));
|
|
213
233
|
}
|
|
214
234
|
function formatVariationQuestionSummary(summary) {
|
|
215
235
|
return `${summary.passed_questions}/${summary.total_questions}`;
|
|
@@ -226,28 +246,28 @@ function printWorkflowVariationSummary(summaries) {
|
|
|
226
246
|
console.log(` | ${summary.variation} | ${summary.kind} | \`${formatVariationQuestionSummary(summary)}\` | ${summary.passed ? "pass" : "fail"} |`);
|
|
227
247
|
}
|
|
228
248
|
}
|
|
229
|
-
function
|
|
249
|
+
function printCompiledLocalWorkflowOwnership(compiledPath, bestVariation) {
|
|
230
250
|
if (bestVariation <= 1)
|
|
231
251
|
return;
|
|
232
252
|
console.log();
|
|
233
|
-
console.log(chalk.dim(` Best workflow variation now lives in: ${
|
|
234
|
-
console.log(chalk.dim("
|
|
235
|
-
console.log(chalk.dim(" Recompiling this existing
|
|
253
|
+
console.log(chalk.dim(` Best workflow variation now lives in: ${workflowPackagePathForCompiled(compiledPath)}`));
|
|
254
|
+
console.log(chalk.dim(" The project-level workflow selection was not changed automatically."));
|
|
255
|
+
console.log(chalk.dim(" Recompiling this existing compiled dataset reuses that local workflow draft."));
|
|
236
256
|
}
|
|
237
|
-
export async function
|
|
257
|
+
export async function runConfiguredCompiledCompile(options) {
|
|
238
258
|
const preserveStageShells = options.preserveStageShells ?? "on-failure";
|
|
239
|
-
const configuredMaxAttempts =
|
|
240
|
-
const maxLoops =
|
|
259
|
+
const configuredMaxAttempts = resolveDatasetCompileMaxAttempts(options.compiledConfig ?? { max_attempts: undefined }, options.maxAttemptsOverride);
|
|
260
|
+
const maxLoops = resolveDatasetCompileMaxLoops(options.compiledConfig ?? { max_loops: undefined }, options.maxLoopsOverride);
|
|
241
261
|
const maxAttempts = configuredMaxAttempts ?? (maxLoops != null ? 1 : null);
|
|
242
262
|
const loopEnabled = maxAttempts != null;
|
|
243
|
-
const checks = options.
|
|
263
|
+
const checks = options.compiledConfig?.checks ?? [];
|
|
244
264
|
if (loopEnabled && checks.length === 0) {
|
|
245
265
|
console.log(chalk.yellow(" Retry and self-improving modes need saved truth checks. Building once without those loops."));
|
|
246
266
|
}
|
|
247
267
|
else if (loopEnabled) {
|
|
248
268
|
if (maxAttempts > 1) {
|
|
249
269
|
console.log(chalk.dim(` Retry mode: up to ${maxAttempts} compile attempts.`));
|
|
250
|
-
console.log(chalk.dim(" Interf Compiler will rerun the same workflow variation, test the
|
|
270
|
+
console.log(chalk.dim(" Interf Compiler will rerun the same workflow variation, test the compiled dataset, and stop early if it passes."));
|
|
251
271
|
}
|
|
252
272
|
else {
|
|
253
273
|
console.log(chalk.dim(" Compile mode: 1 attempt per workflow variation."));
|
|
@@ -258,16 +278,21 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
258
278
|
}
|
|
259
279
|
}
|
|
260
280
|
if (!loopEnabled || checks.length === 0) {
|
|
261
|
-
const result = await
|
|
281
|
+
const result = await compileCompiledWithReporter(options.executor, options.compiledPath, {
|
|
262
282
|
preserveStageShells,
|
|
263
283
|
});
|
|
264
284
|
if (!result.ok) {
|
|
265
285
|
process.exitCode = 1;
|
|
266
|
-
printCompileFailure(options.
|
|
267
|
-
printStageShellReviewHint(options.
|
|
286
|
+
printCompileFailure(options.compiledPath, result.failedStage);
|
|
287
|
+
printStageShellReviewHint(options.compiledPath, preserveStageShells, "failure");
|
|
268
288
|
return false;
|
|
269
289
|
}
|
|
270
|
-
printStageShellReviewHint(options.
|
|
290
|
+
printStageShellReviewHint(options.compiledPath, preserveStageShells, "success");
|
|
291
|
+
printPostCompileNextStep({
|
|
292
|
+
compiledPath: options.compiledPath,
|
|
293
|
+
checks: checks.length,
|
|
294
|
+
testedDuringCompile: false,
|
|
295
|
+
});
|
|
271
296
|
return true;
|
|
272
297
|
}
|
|
273
298
|
let bestOutcome = null;
|
|
@@ -277,8 +302,9 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
277
302
|
const previousVariations = [];
|
|
278
303
|
const improvementRunId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
279
304
|
try {
|
|
280
|
-
const
|
|
281
|
-
name: readInterfConfig(options.
|
|
305
|
+
const compiledConfig = options.compiledConfig ?? {
|
|
306
|
+
name: readInterfConfig(options.compiledPath)?.name ?? "compiled",
|
|
307
|
+
path: ".",
|
|
282
308
|
checks: [],
|
|
283
309
|
};
|
|
284
310
|
const adoptVariationBest = (variation, result) => {
|
|
@@ -300,9 +326,9 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
300
326
|
const baseline = await runWorkflowVariation({
|
|
301
327
|
executor: options.executor,
|
|
302
328
|
testExecutor: options.testExecutor,
|
|
303
|
-
|
|
329
|
+
compiledPath: options.compiledPath,
|
|
304
330
|
sourcePath: options.sourcePath,
|
|
305
|
-
|
|
331
|
+
compiledConfig,
|
|
306
332
|
executionProfile: options.executionProfile,
|
|
307
333
|
preserveStageShells,
|
|
308
334
|
maxAttempts,
|
|
@@ -322,6 +348,11 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
322
348
|
}));
|
|
323
349
|
if (baseline.passed) {
|
|
324
350
|
printWorkflowVariationSummary(previousVariations);
|
|
351
|
+
printPostCompileNextStep({
|
|
352
|
+
compiledPath: options.compiledPath,
|
|
353
|
+
checks: checks.length,
|
|
354
|
+
testedDuringCompile: true,
|
|
355
|
+
});
|
|
325
356
|
return true;
|
|
326
357
|
}
|
|
327
358
|
if (maxLoops != null) {
|
|
@@ -330,9 +361,9 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
330
361
|
console.log(chalk.dim(` Self-improving loop ${loopIndex}/${maxLoops}: reviewing failed runs and editing the workflow.`));
|
|
331
362
|
const improvement = await runWorkflowImprovementLoop({
|
|
332
363
|
executor: options.executor,
|
|
333
|
-
|
|
334
|
-
workflowId:
|
|
335
|
-
|
|
364
|
+
compiledPath: options.compiledPath,
|
|
365
|
+
workflowId: resolveRequiredCompiledWorkflowFromConfig(readInterfConfig(options.compiledPath), `.interf/interf.json for ${options.compiledPath}`),
|
|
366
|
+
compiledConfig,
|
|
336
367
|
runId: improvementRunId,
|
|
337
368
|
loopIndex,
|
|
338
369
|
maxLoops,
|
|
@@ -352,9 +383,9 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
352
383
|
const variation = await runWorkflowVariation({
|
|
353
384
|
executor: options.executor,
|
|
354
385
|
testExecutor: options.testExecutor,
|
|
355
|
-
|
|
386
|
+
compiledPath: options.compiledPath,
|
|
356
387
|
sourcePath: options.sourcePath,
|
|
357
|
-
|
|
388
|
+
compiledConfig,
|
|
358
389
|
executionProfile: options.executionProfile,
|
|
359
390
|
preserveStageShells,
|
|
360
391
|
maxAttempts,
|
|
@@ -375,30 +406,35 @@ export async function runConfiguredWorkspaceCompile(options) {
|
|
|
375
406
|
}));
|
|
376
407
|
if (variation.passed) {
|
|
377
408
|
printWorkflowVariationSummary(previousVariations);
|
|
378
|
-
|
|
409
|
+
printCompiledLocalWorkflowOwnership(options.compiledPath, bestVariation);
|
|
410
|
+
printPostCompileNextStep({
|
|
411
|
+
compiledPath: options.compiledPath,
|
|
412
|
+
checks: checks.length,
|
|
413
|
+
testedDuringCompile: true,
|
|
414
|
+
});
|
|
379
415
|
return true;
|
|
380
416
|
}
|
|
381
417
|
}
|
|
382
418
|
}
|
|
383
419
|
if (bestSnapshotPath && bestVariation > 0) {
|
|
384
|
-
|
|
420
|
+
restoreCompiledBestState(bestSnapshotPath, options.compiledPath);
|
|
385
421
|
console.log();
|
|
386
|
-
console.log(chalk.dim(` Restored the best-performing compiled
|
|
422
|
+
console.log(chalk.dim(` Restored the best-performing compiled dataset from variation ${bestVariation}, attempt ${bestAttempt}.`));
|
|
387
423
|
}
|
|
388
424
|
process.exitCode = 1;
|
|
389
425
|
console.log();
|
|
390
426
|
printWorkflowVariationSummary(previousVariations);
|
|
391
|
-
|
|
427
|
+
printCompiledLocalWorkflowOwnership(options.compiledPath, bestVariation);
|
|
392
428
|
if (maxLoops != null) {
|
|
393
|
-
console.log(chalk.red(`
|
|
429
|
+
console.log(chalk.red(` Compiled did not pass after ${maxAttempts} attempt${maxAttempts === 1 ? "" : "s"} per variation and ${maxLoops} workflow-improvement loop${maxLoops === 1 ? "" : "s"}.`));
|
|
394
430
|
}
|
|
395
431
|
else {
|
|
396
|
-
console.log(chalk.red(`
|
|
432
|
+
console.log(chalk.red(` Compiled did not pass within ${maxAttempts} attempt${maxAttempts === 1 ? "" : "s"}.`));
|
|
397
433
|
}
|
|
398
434
|
if (bestOutcome) {
|
|
399
435
|
console.log(chalk.dim(` Best attempt truth-check pass rate: ${questionPassRate(bestOutcome)}%.`));
|
|
400
436
|
}
|
|
401
|
-
printStageShellReviewHint(options.
|
|
437
|
+
printStageShellReviewHint(options.compiledPath, preserveStageShells, "failure");
|
|
402
438
|
return false;
|
|
403
439
|
}
|
|
404
440
|
finally {
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { SourceDatasetConfig } from "../lib/schema.js";
|
|
2
2
|
import type { CommandModule } from "yargs";
|
|
3
|
-
export {
|
|
3
|
+
export { runConfiguredCompiledCompile } from "./compile-controller.js";
|
|
4
4
|
export declare const compileCommand: CommandModule;
|
|
5
5
|
export declare function runCompileCommand(argv?: Record<string, unknown>): Promise<void>;
|
|
6
6
|
export declare function resolveInteractiveCompileLoopOverrides(options: {
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
compiledPath: string;
|
|
8
|
+
compiledConfig: SourceDatasetConfig | null;
|
|
9
9
|
maxAttemptsOverride: number | null;
|
|
10
10
|
maxLoopsOverride: number | null;
|
|
11
11
|
}): Promise<{
|
|
12
|
-
|
|
12
|
+
compiledConfig: SourceDatasetConfig | null;
|
|
13
13
|
maxAttemptsOverride: number | null;
|
|
14
14
|
maxLoopsOverride: number | null;
|
|
15
15
|
} | null>;
|