@interf/compiler 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -69
- package/builtin-workflows/interf/README.md +6 -6
- package/builtin-workflows/interf/compile/stages/shape/SKILL.md +7 -7
- package/builtin-workflows/interf/compile/stages/structure/SKILL.md +2 -2
- package/builtin-workflows/interf/compile/stages/summarize/SKILL.md +1 -1
- package/builtin-workflows/interf/{workspace.schema.json → compiled.schema.json} +5 -5
- package/builtin-workflows/interf/improve/SKILL.md +3 -3
- package/builtin-workflows/interf/use/query/SKILL.md +2 -2
- package/builtin-workflows/interf/workflow.json +42 -31
- package/dist/commands/check-draft.d.ts +19 -0
- package/dist/commands/check-draft.js +110 -0
- package/dist/commands/compile-controller.d.ts +4 -4
- package/dist/commands/compile-controller.js +117 -81
- package/dist/commands/compile.d.ts +5 -5
- package/dist/commands/compile.js +61 -62
- package/dist/commands/compiled-flow.d.ts +23 -0
- package/dist/commands/compiled-flow.js +112 -0
- package/dist/commands/create-workflow-wizard.d.ts +3 -3
- package/dist/commands/create-workflow-wizard.js +11 -11
- package/dist/commands/create.d.ts +2 -2
- package/dist/commands/create.js +50 -57
- package/dist/commands/default.js +2 -2
- package/dist/commands/executor-flow.d.ts +20 -1
- package/dist/commands/executor-flow.js +67 -7
- package/dist/commands/init.js +242 -289
- package/dist/commands/list.js +14 -10
- package/dist/commands/reset.js +6 -6
- package/dist/commands/source-config-wizard.d.ts +12 -8
- package/dist/commands/source-config-wizard.js +356 -119
- package/dist/commands/status.js +49 -26
- package/dist/commands/test-flow.d.ts +23 -10
- package/dist/commands/test-flow.js +278 -58
- package/dist/commands/test.d.ts +7 -1
- package/dist/commands/test.js +264 -65
- package/dist/commands/verify.js +23 -14
- package/dist/index.d.ts +7 -7
- package/dist/index.js +4 -4
- package/dist/lib/agent-args.js +2 -1
- package/dist/lib/agent-constants.js +1 -1
- package/dist/lib/agent-render.js +4 -4
- package/dist/lib/agent-shells.d.ts +8 -8
- package/dist/lib/agent-shells.js +231 -142
- package/dist/lib/compiled-compile.d.ts +52 -0
- package/dist/lib/compiled-compile.js +274 -0
- package/dist/lib/compiled-home.d.ts +5 -0
- package/dist/lib/compiled-home.js +32 -0
- package/dist/lib/compiled-layout.d.ts +2 -0
- package/dist/lib/compiled-layout.js +60 -0
- package/dist/lib/compiled-paths.d.ts +41 -0
- package/dist/lib/compiled-paths.js +111 -0
- package/dist/lib/{workspace-raw.d.ts → compiled-raw.d.ts} +8 -7
- package/dist/lib/{workspace-raw.js → compiled-raw.js} +16 -14
- package/dist/lib/compiled-reset.d.ts +1 -0
- package/dist/lib/compiled-reset.js +44 -0
- package/dist/lib/compiled-schema.d.ts +27 -0
- package/dist/lib/compiled-schema.js +110 -0
- package/dist/lib/config.d.ts +0 -1
- package/dist/lib/config.js +0 -1
- package/dist/lib/discovery.d.ts +1 -1
- package/dist/lib/discovery.js +3 -3
- package/dist/lib/interf-bootstrap.d.ts +1 -1
- package/dist/lib/interf-bootstrap.js +4 -4
- package/dist/lib/interf-detect.d.ts +10 -10
- package/dist/lib/interf-detect.js +78 -56
- package/dist/lib/interf-scaffold.d.ts +2 -2
- package/dist/lib/interf-scaffold.js +90 -57
- package/dist/lib/interf-workflow-package.d.ts +3 -3
- package/dist/lib/interf-workflow-package.js +30 -30
- package/dist/lib/interf.d.ts +5 -5
- package/dist/lib/interf.js +4 -4
- package/dist/lib/local-workflows.d.ts +4 -4
- package/dist/lib/local-workflows.js +35 -70
- package/dist/lib/obsidian.d.ts +1 -1
- package/dist/lib/parse.js +92 -1
- package/dist/lib/project-paths.d.ts +13 -0
- package/dist/lib/project-paths.js +29 -0
- package/dist/lib/runtime-acceptance.d.ts +7 -1
- package/dist/lib/runtime-acceptance.js +194 -59
- package/dist/lib/runtime-contracts.d.ts +2 -4
- package/dist/lib/runtime-contracts.js +17 -161
- package/dist/lib/runtime-inventory.d.ts +7 -0
- package/dist/lib/runtime-inventory.js +29 -0
- package/dist/lib/runtime-paths.js +5 -5
- package/dist/lib/runtime-prompt.js +9 -6
- package/dist/lib/runtime-reconcile.d.ts +2 -3
- package/dist/lib/runtime-reconcile.js +92 -171
- package/dist/lib/runtime-runs.js +30 -39
- package/dist/lib/runtime-types.d.ts +10 -19
- package/dist/lib/runtime.d.ts +2 -2
- package/dist/lib/runtime.js +1 -1
- package/dist/lib/schema.d.ts +163 -140
- package/dist/lib/schema.js +163 -124
- package/dist/lib/source-config.d.ts +24 -20
- package/dist/lib/source-config.js +154 -116
- package/dist/lib/state-artifacts.d.ts +5 -5
- package/dist/lib/state-artifacts.js +8 -8
- package/dist/lib/state-health.d.ts +4 -4
- package/dist/lib/state-health.js +108 -126
- package/dist/lib/state-io.d.ts +8 -8
- package/dist/lib/state-io.js +77 -50
- package/dist/lib/state-paths.js +5 -5
- package/dist/lib/state-view.d.ts +4 -4
- package/dist/lib/state-view.js +52 -55
- package/dist/lib/state.d.ts +5 -5
- package/dist/lib/state.js +4 -4
- package/dist/lib/summarize-plan.d.ts +3 -2
- package/dist/lib/summarize-plan.js +18 -16
- package/dist/lib/test-execution.js +9 -9
- package/dist/lib/test-matrices.d.ts +3 -3
- package/dist/lib/test-matrices.js +6 -6
- package/dist/lib/test-paths.d.ts +4 -4
- package/dist/lib/test-paths.js +16 -10
- package/dist/lib/test-sandbox.d.ts +1 -1
- package/dist/lib/test-sandbox.js +38 -31
- package/dist/lib/test-targets.d.ts +2 -2
- package/dist/lib/test-targets.js +11 -11
- package/dist/lib/test-types.d.ts +1 -1
- package/dist/lib/test.d.ts +1 -1
- package/dist/lib/test.js +1 -1
- package/dist/lib/util.d.ts +2 -0
- package/dist/lib/util.js +14 -1
- package/dist/lib/validate-compiled.d.ts +27 -0
- package/dist/lib/validate-compiled.js +236 -0
- package/dist/lib/validate-helpers.d.ts +0 -8
- package/dist/lib/validate-helpers.js +0 -30
- package/dist/lib/validate.d.ts +4 -4
- package/dist/lib/validate.js +49 -15
- package/dist/lib/workflow-abi.d.ts +37 -46
- package/dist/lib/workflow-abi.js +51 -76
- package/dist/lib/workflow-definitions.d.ts +11 -11
- package/dist/lib/workflow-definitions.js +36 -53
- package/dist/lib/workflow-helpers.d.ts +2 -3
- package/dist/lib/workflow-helpers.js +9 -13
- package/dist/lib/workflow-improvement.d.ts +3 -3
- package/dist/lib/workflow-improvement.js +48 -48
- package/dist/lib/workflow-review-paths.d.ts +3 -3
- package/dist/lib/workflow-review-paths.js +11 -11
- package/dist/lib/workflow-stage-runner.d.ts +1 -1
- package/dist/lib/workflow-stage-runner.js +8 -8
- package/dist/lib/workflows.d.ts +9 -9
- package/dist/lib/workflows.js +15 -17
- package/package.json +10 -9
- package/dist/commands/workspace-flow.d.ts +0 -23
- package/dist/commands/workspace-flow.js +0 -109
- package/dist/lib/registry.d.ts +0 -16
- package/dist/lib/registry.js +0 -65
- package/dist/lib/validate-workspace.d.ts +0 -121
- package/dist/lib/validate-workspace.js +0 -407
- package/dist/lib/workspace-compile.d.ts +0 -54
- package/dist/lib/workspace-compile.js +0 -476
- package/dist/lib/workspace-home.d.ts +0 -5
- package/dist/lib/workspace-home.js +0 -32
- package/dist/lib/workspace-layout.d.ts +0 -2
- package/dist/lib/workspace-layout.js +0 -60
- package/dist/lib/workspace-paths.d.ts +0 -41
- package/dist/lib/workspace-paths.js +0 -107
- package/dist/lib/workspace-reset.d.ts +0 -1
- package/dist/lib/workspace-reset.js +0 -43
- package/dist/lib/workspace-schema.d.ts +0 -17
- package/dist/lib/workspace-schema.js +0 -74
package/dist/commands/test.js
CHANGED
|
@@ -1,21 +1,41 @@
|
|
|
1
1
|
import chalk from "chalk";
|
|
2
|
+
import * as p from "@clack/prompts";
|
|
2
3
|
import { detectInterf, resolveSourceControlPath, } from "../lib/interf.js";
|
|
3
|
-
import {
|
|
4
|
+
import { fingerprintTruthChecks, sourceDatasetConfigFromInterfConfig, } from "../lib/source-config.js";
|
|
4
5
|
import { addExecutionProfileOptions, executionProfileFromArgv, } from "../lib/execution-profile.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
6
|
+
import { chooseCompiledConfigToBuild, findBuiltCompiledPath, findSavedCompiledConfig, } from "./compiled-flow.js";
|
|
7
|
+
import { printAgentTestFailures, printAgentTestMatrix, printSavedTestComparisonState, readSavedTestComparison, runSavedRawTest, runSavedCompiledTest, saveTestComparisonRun, } from "./test-flow.js";
|
|
8
|
+
import { listRunAgentOptions, promptForTestAgents, resolveNamedLocalExecutor, resolveOrConfigureLocalExecutor, } from "./executor-flow.js";
|
|
9
|
+
export function resolveConfiguredDatasetSelection(options) {
|
|
10
|
+
if (options.hintedDatasetConfig &&
|
|
11
|
+
(!options.requestedDatasetName || options.hintedDatasetConfig.name === options.requestedDatasetName)) {
|
|
12
|
+
return options.hintedDatasetConfig;
|
|
13
|
+
}
|
|
14
|
+
if (!options.requestedDatasetName) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
return findSavedCompiledConfig(options.sourcePath, options.requestedDatasetName);
|
|
18
|
+
}
|
|
7
19
|
export const testCommand = {
|
|
8
20
|
command: "test",
|
|
9
|
-
describe: "Compare
|
|
21
|
+
describe: "Compare files as-is and a compiled dataset on saved truth checks",
|
|
10
22
|
builder: (yargs) => addExecutionProfileOptions(yargs)
|
|
23
|
+
.option("dataset", {
|
|
24
|
+
type: "string",
|
|
25
|
+
describe: "Dataset id to test when this project has more than one dataset",
|
|
26
|
+
})
|
|
11
27
|
.option("target", {
|
|
12
|
-
choices: ["both", "raw", "
|
|
13
|
-
describe: "Test
|
|
28
|
+
choices: ["both", "raw", "compiled"],
|
|
29
|
+
describe: "Test files as-is, the compiled dataset, or both. Default: both when a compiled dataset exists, otherwise files as-is.",
|
|
14
30
|
})
|
|
15
31
|
.option("keep-sandboxes", {
|
|
16
32
|
type: "boolean",
|
|
17
33
|
default: false,
|
|
18
|
-
describe: "Keep every
|
|
34
|
+
describe: "Keep every test sandbox for review instead of pruning successful ones",
|
|
35
|
+
})
|
|
36
|
+
.option("agents", {
|
|
37
|
+
type: "string",
|
|
38
|
+
describe: "Comma-separated local agents to use for this test, or `all`. Example: --agents claude-code,codex",
|
|
19
39
|
}),
|
|
20
40
|
handler: async (argv) => {
|
|
21
41
|
await runTestCommand(argv);
|
|
@@ -27,82 +47,261 @@ function readSandboxRetentionMode(argv) {
|
|
|
27
47
|
}
|
|
28
48
|
function readRequestedMode(argv) {
|
|
29
49
|
const value = argv.target;
|
|
30
|
-
return value === "raw" || value === "
|
|
50
|
+
return value === "raw" || value === "compiled" || value === "both" ? value : null;
|
|
51
|
+
}
|
|
52
|
+
async function promptTestMode(hasBuiltCompiled) {
|
|
53
|
+
if (!hasBuiltCompiled) {
|
|
54
|
+
return "raw";
|
|
55
|
+
}
|
|
56
|
+
const selected = await p.select({
|
|
57
|
+
message: "What do you want to test?",
|
|
58
|
+
options: [
|
|
59
|
+
{
|
|
60
|
+
value: "both",
|
|
61
|
+
label: "Compare files-as-is and compiled dataset (Recommended)",
|
|
62
|
+
hint: "Measure whether compilation helps on the saved truth checks",
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
value: "raw",
|
|
66
|
+
label: "Files-as-is only",
|
|
67
|
+
hint: "Measure the current baseline on the dataset files",
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
value: "compiled",
|
|
71
|
+
label: "Compiled dataset only",
|
|
72
|
+
hint: "Measure the current compiled dataset on the saved truth checks",
|
|
73
|
+
},
|
|
74
|
+
],
|
|
75
|
+
});
|
|
76
|
+
if (p.isCancel(selected))
|
|
77
|
+
return null;
|
|
78
|
+
return selected;
|
|
79
|
+
}
|
|
80
|
+
function parseRequestedAgents(argv) {
|
|
81
|
+
const raw = argv.agents;
|
|
82
|
+
if (typeof raw !== "string" || raw.trim().length === 0)
|
|
83
|
+
return null;
|
|
84
|
+
if (raw.trim() === "all") {
|
|
85
|
+
return listRunAgentOptions().map((agent) => agent.name);
|
|
86
|
+
}
|
|
87
|
+
return raw
|
|
88
|
+
.split(",")
|
|
89
|
+
.map((value) => value.trim())
|
|
90
|
+
.filter((value) => value.length > 0);
|
|
91
|
+
}
|
|
92
|
+
async function resolveSelectedTestAgents(options) {
|
|
93
|
+
const requestedAgents = parseRequestedAgents(options.argv);
|
|
94
|
+
if (requestedAgents && requestedAgents.length > 0) {
|
|
95
|
+
const selected = [];
|
|
96
|
+
for (const agentName of requestedAgents) {
|
|
97
|
+
const resolved = resolveNamedLocalExecutor(agentName, {
|
|
98
|
+
executionProfile: options.executionProfile,
|
|
99
|
+
});
|
|
100
|
+
if (!resolved.executor) {
|
|
101
|
+
process.exitCode = 1;
|
|
102
|
+
console.log(chalk.red(` ${resolved.error ?? `Could not resolve local agent \"${agentName}\".`}`));
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
const option = listRunAgentOptions().find((agent) => agent.name === agentName);
|
|
106
|
+
selected.push({
|
|
107
|
+
name: agentName,
|
|
108
|
+
label: option?.displayName ?? agentName,
|
|
109
|
+
executor: resolved.executor,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return selected;
|
|
113
|
+
}
|
|
114
|
+
if (process.stdin.isTTY && process.stdout.isTTY) {
|
|
115
|
+
const prompted = await promptForTestAgents();
|
|
116
|
+
if (prompted.cancelled)
|
|
117
|
+
return null;
|
|
118
|
+
if (prompted.agents.length > 0) {
|
|
119
|
+
const selected = [];
|
|
120
|
+
for (const agent of prompted.agents) {
|
|
121
|
+
const resolved = resolveNamedLocalExecutor(agent.name, {
|
|
122
|
+
executionProfile: options.executionProfile,
|
|
123
|
+
});
|
|
124
|
+
if (!resolved.executor) {
|
|
125
|
+
process.exitCode = 1;
|
|
126
|
+
console.log(chalk.red(` ${resolved.error ?? `Could not resolve local agent \"${agent.name}\".`}`));
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
selected.push({
|
|
130
|
+
name: agent.name,
|
|
131
|
+
label: agent.displayName,
|
|
132
|
+
executor: resolved.executor,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return selected;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const fallback = await resolveOrConfigureLocalExecutor({
|
|
139
|
+
preflight: true,
|
|
140
|
+
executionProfile: options.executionProfile,
|
|
141
|
+
purpose: "test",
|
|
142
|
+
});
|
|
143
|
+
if (!fallback.executor) {
|
|
144
|
+
if (fallback.cancelled)
|
|
145
|
+
return null;
|
|
146
|
+
process.exitCode = 1;
|
|
147
|
+
console.log(chalk.red(fallback.error ?? "No local agent detected."));
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
return [
|
|
151
|
+
{
|
|
152
|
+
name: fallback.executor.name,
|
|
153
|
+
label: fallback.executor.displayName,
|
|
154
|
+
executor: fallback.executor,
|
|
155
|
+
},
|
|
156
|
+
];
|
|
157
|
+
}
|
|
158
|
+
async function runModeForAgent(options) {
|
|
159
|
+
const rawOutcome = options.mode === "raw" || options.mode === "both"
|
|
160
|
+
? await runSavedRawTest({
|
|
161
|
+
sourcePath: options.sourcePath,
|
|
162
|
+
datasetConfig: options.datasetConfig,
|
|
163
|
+
executor: options.executor,
|
|
164
|
+
preserveSandboxes: options.preserveSandboxes,
|
|
165
|
+
runSuffix: options.executor.name,
|
|
166
|
+
})
|
|
167
|
+
: null;
|
|
168
|
+
const compiledOutcome = options.mode === "compiled" || options.mode === "both"
|
|
169
|
+
? await runSavedCompiledTest({
|
|
170
|
+
sourcePath: options.sourcePath,
|
|
171
|
+
datasetConfig: options.datasetConfig,
|
|
172
|
+
compiledPath: options.builtCompiledPath,
|
|
173
|
+
executor: options.executor,
|
|
174
|
+
preserveSandboxes: options.preserveSandboxes,
|
|
175
|
+
runSuffix: options.executor.name,
|
|
176
|
+
})
|
|
177
|
+
: null;
|
|
178
|
+
return {
|
|
179
|
+
agentLabel: options.executor.displayName,
|
|
180
|
+
rawOutcome,
|
|
181
|
+
compiledOutcome,
|
|
182
|
+
};
|
|
31
183
|
}
|
|
32
184
|
export async function runTestCommand(argv = {}) {
|
|
33
185
|
const cwd = process.cwd();
|
|
34
186
|
const detected = detectInterf(cwd);
|
|
35
|
-
const sourcePath =
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
187
|
+
const sourcePath = typeof argv.sourcePath === "string" && argv.sourcePath.trim().length > 0
|
|
188
|
+
? argv.sourcePath.trim()
|
|
189
|
+
: (detected ? resolveSourceControlPath(detected.path) : cwd);
|
|
190
|
+
const requestedDatasetName = typeof argv.dataset === "string" && argv.dataset.trim().length > 0
|
|
191
|
+
? argv.dataset.trim()
|
|
192
|
+
: null;
|
|
193
|
+
const hintedDatasetConfig = (argv.datasetConfig ?? null);
|
|
194
|
+
const selectedCompiled = detected
|
|
195
|
+
? (() => {
|
|
196
|
+
const targetName = requestedDatasetName ?? hintedDatasetConfig?.name ?? detected.config.name;
|
|
197
|
+
return resolveConfiguredDatasetSelection({
|
|
198
|
+
sourcePath,
|
|
199
|
+
requestedDatasetName: targetName,
|
|
200
|
+
hintedDatasetConfig,
|
|
201
|
+
}) ?? (targetName === detected.config.name ? sourceDatasetConfigFromInterfConfig(detected.config) : null);
|
|
202
|
+
})()
|
|
203
|
+
: requestedDatasetName || hintedDatasetConfig
|
|
204
|
+
? resolveConfiguredDatasetSelection({
|
|
205
|
+
sourcePath,
|
|
206
|
+
requestedDatasetName,
|
|
207
|
+
hintedDatasetConfig,
|
|
208
|
+
})
|
|
209
|
+
: await chooseCompiledConfigToBuild({
|
|
210
|
+
sourcePath,
|
|
211
|
+
selectMessage: "Which dataset do you want to test?",
|
|
212
|
+
});
|
|
213
|
+
if (selectedCompiled === undefined)
|
|
214
|
+
return false;
|
|
215
|
+
if (!selectedCompiled) {
|
|
45
216
|
process.exitCode = 1;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
217
|
+
if (requestedDatasetName) {
|
|
218
|
+
console.log(chalk.red(` Dataset "${requestedDatasetName}" is not set up in this project.`));
|
|
219
|
+
console.log(chalk.dim(" Run `interf list` to see the saved datasets."));
|
|
220
|
+
}
|
|
221
|
+
else {
|
|
222
|
+
console.log(chalk.red(" No datasets are set up for this folder yet."));
|
|
223
|
+
console.log(chalk.dim(" Start with `interf` or `interf init`."));
|
|
224
|
+
}
|
|
225
|
+
return false;
|
|
49
226
|
}
|
|
50
|
-
if (
|
|
227
|
+
if (selectedCompiled.checks.length === 0) {
|
|
51
228
|
process.exitCode = 1;
|
|
52
|
-
console.log(chalk.red(`
|
|
53
|
-
console.log(chalk.dim(" Run `interf
|
|
54
|
-
|
|
229
|
+
console.log(chalk.red(` Dataset "${selectedCompiled.name}" does not have any saved truth checks yet.`));
|
|
230
|
+
console.log(chalk.dim(" Run `interf`, edit this dataset, and add a few truth checks first."));
|
|
231
|
+
console.log(chalk.dim(" Then rerun `interf test`."));
|
|
232
|
+
return false;
|
|
55
233
|
}
|
|
56
234
|
const executionProfile = executionProfileFromArgv(argv);
|
|
57
235
|
const preserveSandboxes = readSandboxRetentionMode(argv);
|
|
58
|
-
const
|
|
59
|
-
|
|
236
|
+
const builtCompiledPath = detected?.config.name === selectedCompiled.name
|
|
237
|
+
? detected.path
|
|
238
|
+
: findBuiltCompiledPath(sourcePath, selectedCompiled.name);
|
|
239
|
+
const hasBuiltCompiled = Boolean(builtCompiledPath);
|
|
60
240
|
const requestedMode = readRequestedMode(argv);
|
|
61
|
-
const selectedMode = requestedMode ?? (
|
|
62
|
-
|
|
241
|
+
const selectedMode = requestedMode ?? (process.stdin.isTTY && process.stdout.isTTY
|
|
242
|
+
? await promptTestMode(hasBuiltCompiled)
|
|
243
|
+
: (hasBuiltCompiled ? "both" : "raw"));
|
|
244
|
+
if (!selectedMode)
|
|
245
|
+
return false;
|
|
246
|
+
if (selectedMode === "compiled" && !hasBuiltCompiled) {
|
|
63
247
|
process.exitCode = 1;
|
|
64
|
-
console.log(chalk.red(`
|
|
248
|
+
console.log(chalk.red(` Dataset "${selectedCompiled.name}" is not compiled yet.`));
|
|
65
249
|
console.log(chalk.dim(" Run `interf compile` first."));
|
|
66
|
-
return;
|
|
250
|
+
return false;
|
|
67
251
|
}
|
|
68
|
-
if (selectedMode === "both" && !
|
|
69
|
-
console.log(chalk.dim(`
|
|
252
|
+
if (selectedMode === "both" && !hasBuiltCompiled) {
|
|
253
|
+
console.log(chalk.dim(` Dataset "${selectedCompiled.name}" is not compiled yet. Running the files as-is only.`));
|
|
70
254
|
}
|
|
71
|
-
const mode = selectedMode === "both" && !
|
|
255
|
+
const mode = selectedMode === "both" && !hasBuiltCompiled
|
|
72
256
|
? "raw"
|
|
73
257
|
: selectedMode;
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
executionProfile,
|
|
81
|
-
preserveSandboxes,
|
|
82
|
-
})
|
|
83
|
-
: null;
|
|
84
|
-
const workspaceOutcome = mode === "workspace" || mode === "both"
|
|
85
|
-
? await runSavedWorkspaceTest({
|
|
86
|
-
sourcePath,
|
|
87
|
-
workspaceConfig: selectedWorkspace,
|
|
88
|
-
executionProfile,
|
|
89
|
-
workspacePath: artifactWorkspacePath,
|
|
90
|
-
preserveSandboxes,
|
|
91
|
-
})
|
|
92
|
-
: null;
|
|
93
|
-
if ((mode === "workspace" || mode === "both") && !workspaceOutcome) {
|
|
94
|
-
process.exitCode = 1;
|
|
95
|
-
console.log(chalk.red(` Workspace "${selectedWorkspace.name}" is not compiled yet.`));
|
|
96
|
-
console.log(chalk.dim(" Run `interf compile` first."));
|
|
97
|
-
return;
|
|
258
|
+
const selectedAgents = await resolveSelectedTestAgents({
|
|
259
|
+
argv,
|
|
260
|
+
executionProfile,
|
|
261
|
+
});
|
|
262
|
+
if (!selectedAgents || selectedAgents.length === 0) {
|
|
263
|
+
return false;
|
|
98
264
|
}
|
|
99
|
-
const
|
|
265
|
+
const rows = await Promise.all(selectedAgents.map((agent) => runModeForAgent({
|
|
100
266
|
sourcePath,
|
|
101
|
-
|
|
102
|
-
|
|
267
|
+
datasetConfig: selectedCompiled,
|
|
268
|
+
builtCompiledPath: builtCompiledPath ?? null,
|
|
103
269
|
mode,
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
});
|
|
107
|
-
|
|
270
|
+
preserveSandboxes,
|
|
271
|
+
executor: agent.executor,
|
|
272
|
+
})));
|
|
273
|
+
if ((mode === "compiled" || mode === "both") && rows.some((row) => !row.compiledOutcome)) {
|
|
274
|
+
process.exitCode = 1;
|
|
275
|
+
console.log(chalk.red(` Dataset "${selectedCompiled.name}" is not compiled yet.`));
|
|
276
|
+
console.log(chalk.dim(" Run `interf compile` first."));
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
printAgentTestMatrix(rows);
|
|
280
|
+
printAgentTestFailures(rows);
|
|
281
|
+
if (rows.length === 1) {
|
|
282
|
+
const row = rows[0];
|
|
283
|
+
const comparisonRunPath = saveTestComparisonRun({
|
|
284
|
+
sourcePath,
|
|
285
|
+
compiledPath: builtCompiledPath ?? null,
|
|
286
|
+
compiledName: selectedCompiled.name,
|
|
287
|
+
checksFingerprint: fingerprintTruthChecks(selectedCompiled.checks),
|
|
288
|
+
mode,
|
|
289
|
+
rawOutcome: row?.rawOutcome ?? null,
|
|
290
|
+
compiledOutcome: row?.compiledOutcome ?? null,
|
|
291
|
+
});
|
|
292
|
+
const latestComparison = readSavedTestComparison(sourcePath, selectedCompiled.name);
|
|
293
|
+
if (latestComparison) {
|
|
294
|
+
if (mode === "both") {
|
|
295
|
+
console.log();
|
|
296
|
+
console.log(chalk.dim(` Saved summary: ${comparisonRunPath}`));
|
|
297
|
+
}
|
|
298
|
+
else {
|
|
299
|
+
printSavedTestComparisonState(latestComparison, comparisonRunPath);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
console.log();
|
|
305
|
+
console.log(chalk.dim(" Multi-agent compare does not overwrite the dataset's saved latest result."));
|
|
306
|
+
return true;
|
|
108
307
|
}
|
package/dist/commands/verify.js
CHANGED
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
import chalk from "chalk";
|
|
2
2
|
import { detectInterf } from "../lib/interf.js";
|
|
3
|
-
import {
|
|
3
|
+
import { validateCompiledCompile, validateCompiledStage, } from "../lib/validate.js";
|
|
4
4
|
import { validateWorkflowPackage } from "../lib/local-workflows.js";
|
|
5
5
|
function normalizeVerifyCheck(value) {
|
|
6
6
|
if (!value)
|
|
7
7
|
return null;
|
|
8
|
-
if (value === "
|
|
9
|
-
return "
|
|
10
|
-
if (value === "
|
|
11
|
-
return "
|
|
12
|
-
if (value === "compile")
|
|
13
|
-
return "compile";
|
|
8
|
+
if (value === "stage")
|
|
9
|
+
return "stage";
|
|
10
|
+
if (value === "compiled" || value === "compile")
|
|
11
|
+
return "compiled";
|
|
14
12
|
if (value === "workflow")
|
|
15
13
|
return "workflow";
|
|
16
14
|
return null;
|
|
@@ -21,7 +19,11 @@ export const verifyCommand = {
|
|
|
21
19
|
builder: (yargs) => yargs
|
|
22
20
|
.positional("check", {
|
|
23
21
|
type: "string",
|
|
24
|
-
describe: "Check to run (`
|
|
22
|
+
describe: "Check to run (`stage`, `compiled`, or `workflow`)",
|
|
23
|
+
})
|
|
24
|
+
.positional("stageId", {
|
|
25
|
+
type: "string",
|
|
26
|
+
describe: "Stage id when using `verify stage <stage-id>`",
|
|
25
27
|
})
|
|
26
28
|
.option("path", {
|
|
27
29
|
type: "string",
|
|
@@ -53,14 +55,21 @@ export const verifyCommand = {
|
|
|
53
55
|
const detected = detectInterf(process.cwd());
|
|
54
56
|
if (!detected) {
|
|
55
57
|
process.exitCode = 1;
|
|
56
|
-
console.log(chalk.red("Run this from inside a compiled
|
|
58
|
+
console.log(chalk.red("Run this from inside a compiled dataset."));
|
|
57
59
|
return;
|
|
58
60
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
61
|
+
if (check === "compiled") {
|
|
62
|
+
result = validateCompiledCompile(detected.path);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
const stageId = argv.stageId;
|
|
66
|
+
if (!stageId) {
|
|
67
|
+
process.exitCode = 1;
|
|
68
|
+
console.log(chalk.red("`interf verify stage` needs <stage-id>."));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
result = validateCompiledStage(detected.path, stageId);
|
|
72
|
+
}
|
|
64
73
|
}
|
|
65
74
|
if (argv.json) {
|
|
66
75
|
console.log(JSON.stringify(result, null, 2));
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { createCompiled, compileCompiled, runCompiledSummarize, runCompiledCompile, } from "./lib/workflows.js";
|
|
2
2
|
export { createRawTestTarget, listTestSpecs, loadTestSpec, loadTestSpecFromFile, writeTestSpec, listTestTargets, runTargetTests, runTargetTestsWithJudge, runTargetTestsAuto, saveTargetTestRun, } from "./lib/test.js";
|
|
3
|
-
export {
|
|
3
|
+
export { computeCompiledHealth, } from "./lib/state.js";
|
|
4
4
|
export { SOURCE_FOLDER_CONFIG_FILE, loadSourceFolderConfig, buildTestSpecFromSourceFolderConfig, } from "./lib/source-config.js";
|
|
5
|
-
export {
|
|
6
|
-
export { InterfConfigSchema, TestSpecSchema, SourceFolderConfigSchema,
|
|
5
|
+
export { validateCompiledSummarize, validateCompiledCompile, } from "./lib/validate.js";
|
|
6
|
+
export { InterfConfigSchema, TestSpecSchema, SourceFolderConfigSchema, CompiledHealthSchema, } from "./lib/schema.js";
|
|
7
7
|
export type { InterfConfig } from "./lib/interf.js";
|
|
8
8
|
export type { TestCaseExpect, TestCase, TestSpec, TestCheckResult, TestCaseResult, TestTargetResult, TestTargetRun, LoadedTestSpec, TestTargetCandidate, } from "./lib/test.js";
|
|
9
|
-
export type { SourceTruthCheck,
|
|
10
|
-
export type {
|
|
11
|
-
export type {
|
|
9
|
+
export type { SourceTruthCheck, SourceDatasetConfig, SourceFolderConfig, CompiledHealth, } from "./lib/schema.js";
|
|
10
|
+
export type { CompiledStageValidation, CompiledWorkflowValidation, } from "./lib/validate.js";
|
|
11
|
+
export type { CompiledSummarizeResult, CompiledCompileResult, } from "./lib/workflows.js";
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { createCompiled, compileCompiled, runCompiledSummarize, runCompiledCompile, } from "./lib/workflows.js";
|
|
2
2
|
export { createRawTestTarget, listTestSpecs, loadTestSpec, loadTestSpecFromFile, writeTestSpec, listTestTargets, runTargetTests, runTargetTestsWithJudge, runTargetTestsAuto, saveTargetTestRun, } from "./lib/test.js";
|
|
3
|
-
export {
|
|
3
|
+
export { computeCompiledHealth, } from "./lib/state.js";
|
|
4
4
|
export { SOURCE_FOLDER_CONFIG_FILE, loadSourceFolderConfig, buildTestSpecFromSourceFolderConfig, } from "./lib/source-config.js";
|
|
5
|
-
export {
|
|
6
|
-
export { InterfConfigSchema, TestSpecSchema, SourceFolderConfigSchema,
|
|
5
|
+
export { validateCompiledSummarize, validateCompiledCompile, } from "./lib/validate.js";
|
|
6
|
+
export { InterfConfigSchema, TestSpecSchema, SourceFolderConfigSchema, CompiledHealthSchema, } from "./lib/schema.js";
|
package/dist/lib/agent-args.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
const CODEX_SANDBOX_MODE = `work${"space-write"}`;
|
|
1
2
|
export function buildAgentArgs(agent, prompt, executionProfile = {}) {
|
|
2
3
|
if (agent.name === "claude-code") {
|
|
3
4
|
const args = [
|
|
@@ -22,7 +23,7 @@ export function buildAgentArgs(agent, prompt, executionProfile = {}) {
|
|
|
22
23
|
"exec",
|
|
23
24
|
"--json",
|
|
24
25
|
"-s",
|
|
25
|
-
|
|
26
|
+
CODEX_SANDBOX_MODE,
|
|
26
27
|
"--skip-git-repo-check",
|
|
27
28
|
];
|
|
28
29
|
if (executionProfile.model) {
|
|
@@ -22,7 +22,7 @@ export const AGENTS = [
|
|
|
22
22
|
},
|
|
23
23
|
];
|
|
24
24
|
export const CODEX_NOISE_PATTERNS = [
|
|
25
|
-
/codex_core_skills::loader: failed to stat skills entry .*\/(interf-[^ ]+|
|
|
25
|
+
/codex_core_skills::loader: failed to stat skills entry .*\/(interf-[^ ]+|compiled\/[^ ]+) \(symlink\): No such file or directory/,
|
|
26
26
|
/codex_core::codex: failed to load skill .*\/gstack\/SKILL\.md: invalid description: exceeds maximum length of 1024 characters/,
|
|
27
27
|
];
|
|
28
28
|
export const VISIBLE_STATUS_PREFIXES = ["STATUS:", "DONE:", "BLOCKED:", "ERROR:"];
|
package/dist/lib/agent-render.js
CHANGED
|
@@ -167,14 +167,14 @@ function summarizeToolUse(name, input) {
|
|
|
167
167
|
const target = summarizeToolTarget(input);
|
|
168
168
|
switch (name) {
|
|
169
169
|
case "Read":
|
|
170
|
-
return target ? `reading ${target}` : "reading
|
|
170
|
+
return target ? `reading ${target}` : "reading compiled files";
|
|
171
171
|
case "Write":
|
|
172
|
-
return target ? `writing ${target}` : "writing
|
|
172
|
+
return target ? `writing ${target}` : "writing compiled files";
|
|
173
173
|
case "Edit":
|
|
174
|
-
return target ? `editing ${target}` : "editing
|
|
174
|
+
return target ? `editing ${target}` : "editing compiled files";
|
|
175
175
|
case "Glob":
|
|
176
176
|
case "Grep":
|
|
177
|
-
return target ? `${name.toLowerCase()} on ${target}` : `${name.toLowerCase()}ing
|
|
177
|
+
return target ? `${name.toLowerCase()} on ${target}` : `${name.toLowerCase()}ing compiled files`;
|
|
178
178
|
case "Bash":
|
|
179
179
|
return target ? `running ${target}` : "running a local helper";
|
|
180
180
|
default:
|
|
@@ -8,26 +8,26 @@ export interface NativeStageDefinition {
|
|
|
8
8
|
reads: WorkflowZoneId[];
|
|
9
9
|
writes: WorkflowZoneId[];
|
|
10
10
|
}
|
|
11
|
-
export declare function
|
|
11
|
+
export declare function renderCompiledAgents(compiledPath: string, name: string, workflowId: string, about?: string, options?: {
|
|
12
12
|
workflowOriginSelected?: string | null;
|
|
13
13
|
workflowLocalDraft?: boolean;
|
|
14
14
|
}): string;
|
|
15
|
-
export declare function
|
|
16
|
-
export declare function syncStageExecutionShellWrites(
|
|
15
|
+
export declare function renderCompiledQuerySkill(): string;
|
|
16
|
+
export declare function syncStageExecutionShellWrites(compiledPath: string, shellRoot: string, stage: NativeStageDefinition, writeArtifacts?: readonly string[]): void;
|
|
17
17
|
export declare function renderClaudeBootstrap(content: string): string;
|
|
18
|
-
export declare function pruneStageExecutionShells(
|
|
19
|
-
export declare function
|
|
18
|
+
export declare function pruneStageExecutionShells(compiledPath: string): void;
|
|
19
|
+
export declare function projectCompiledQueryShell(compiledPath: string, compiledName: string, workflowId: string, about?: string, options?: {
|
|
20
20
|
workflowOriginSelected?: string | null;
|
|
21
21
|
workflowLocalDraft?: boolean;
|
|
22
22
|
}): boolean;
|
|
23
23
|
export declare function projectRawTestQueryShell(rootPath: string): boolean;
|
|
24
|
-
export declare function createStageExecutionShell(
|
|
24
|
+
export declare function createStageExecutionShell(compiledPath: string, compiledName: string, workflowId: string, stage: NativeStageDefinition, writeArtifacts?: readonly string[]): {
|
|
25
25
|
rootPath: string;
|
|
26
26
|
};
|
|
27
27
|
export declare function freezeStageExecutionShell(rootPath: string): string | null;
|
|
28
28
|
export declare function createWorkflowImprovementShell(options: {
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
compiledPath: string;
|
|
30
|
+
compiledName: string;
|
|
31
31
|
workflowId: string;
|
|
32
32
|
runId: string;
|
|
33
33
|
loopIndex: number;
|