@interf/compiler 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -72
- package/builtin-workflows/interf/README.md +4 -4
- package/builtin-workflows/interf/compile/stages/shape/SKILL.md +4 -4
- package/builtin-workflows/interf/improve/SKILL.md +1 -1
- package/builtin-workflows/interf/use/query/SKILL.md +1 -1
- package/builtin-workflows/interf/workflow.json +4 -4
- package/builtin-workflows/interf/workflow.schema.json +1 -1
- package/dist/cli/commands/compile-controller.js +5 -5
- package/dist/cli/commands/compile.js +1 -1
- package/dist/cli/commands/create-workflow-wizard.d.ts +6 -0
- package/dist/cli/commands/create-workflow-wizard.js +57 -5
- package/dist/cli/commands/create.js +21 -11
- package/dist/cli/commands/default.js +2 -1
- package/dist/cli/commands/init.js +65 -54
- package/dist/cli/commands/source-config-wizard.d.ts +0 -1
- package/dist/cli/commands/source-config-wizard.js +1 -4
- package/dist/cli/commands/status.js +3 -3
- package/dist/cli/commands/test-flow.js +9 -7
- package/dist/cli/commands/test.js +7 -7
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/packages/agents/lib/shells.js +22 -19
- package/dist/packages/compiler/compiled-paths.js +1 -1
- package/dist/packages/compiler/runtime-prompt.js +1 -1
- package/dist/packages/project-model/interf-detect.js +24 -10
- package/dist/packages/project-model/interf-scaffold.d.ts +1 -0
- package/dist/packages/project-model/interf-scaffold.js +30 -11
- package/dist/packages/project-model/interf.d.ts +1 -1
- package/dist/packages/project-model/interf.js +1 -1
- package/dist/packages/project-model/project-paths.d.ts +3 -1
- package/dist/packages/project-model/project-paths.js +6 -2
- package/dist/packages/project-model/source-config.d.ts +5 -0
- package/dist/packages/project-model/source-config.js +60 -7
- package/dist/packages/testing/test-paths.js +6 -3
- package/dist/packages/workflow-authoring/workflow-authoring.js +4 -2
- package/dist/packages/workflow-authoring/workflow-edit-session.js +5 -2
- package/dist/packages/workflow-package/context-interface.js +1 -1
- package/dist/packages/workflow-package/workflow-review-paths.js +5 -1
- package/package.json +2 -4
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import chalk from "chalk";
|
|
2
2
|
import * as p from "@clack/prompts";
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
|
-
import { detectInterf, readInterfConfig, resolveSourceControlPath, } from "../../packages/project-model/interf.js";
|
|
5
|
-
import {
|
|
4
|
+
import { detectInterf, ensurePortableContextScaffold, readInterfConfig, resolveSourceControlPath, } from "../../packages/project-model/interf.js";
|
|
5
|
+
import { SOURCE_FOLDER_CONFIG_PATH, migrateLegacySourceFolderConfig, syncCompiledInterfConfigFromSourceDatasetConfig, upsertSourceDatasetConfig, } from "../../packages/project-model/source-config.js";
|
|
6
6
|
import { DEFAULT_COMPILED_NAME, describeCompileLoopSelection, promptSingleCompiledConfig, } from "./source-config-wizard.js";
|
|
7
|
-
import {
|
|
7
|
+
import { chooseOrCreateCompiledWorkflowForDataset, createWorkflowWizard, } from "./create-workflow-wizard.js";
|
|
8
8
|
import { seedLocalDefaultWorkflow } from "../../packages/workflow-package/local-workflows.js";
|
|
9
9
|
import { findBuiltCompiledPath, findSavedCompiledConfig, listSavedCompiledEntries, } from "./compiled-flow.js";
|
|
10
10
|
import { readCurrentSavedTestComparison, } from "./test-flow.js";
|
|
@@ -47,7 +47,7 @@ function printDatasetSummary(options) {
|
|
|
47
47
|
p.log.info(describeSavedQuestions(options.dataset));
|
|
48
48
|
p.log.info(options.built ? "Portable context is available." : "Portable context has not been built yet.");
|
|
49
49
|
if (options.latestComparison?.raw && options.latestComparison?.compiled) {
|
|
50
|
-
p.log.info(`Latest saved comparison: source files ${options.latestComparison.raw.passed_cases}/${options.latestComparison.raw.total_cases}, portable context
|
|
50
|
+
p.log.info(`Latest saved comparison: source files ${options.latestComparison.raw.passed_cases}/${options.latestComparison.raw.total_cases}, portable context ${options.latestComparison.compiled.passed_cases}/${options.latestComparison.compiled.total_cases}.`);
|
|
51
51
|
}
|
|
52
52
|
else if (options.latestComparison?.raw) {
|
|
53
53
|
p.log.info(`Latest saved source-files run: ${options.latestComparison.raw.passed_cases}/${options.latestComparison.raw.total_cases}.`);
|
|
@@ -67,34 +67,44 @@ async function promptDatasetAction(dataset, built, latestComparison) {
|
|
|
67
67
|
const hasSavedRawBaseline = Boolean(latestComparison?.raw);
|
|
68
68
|
const latestRawPasses = latestComparison?.raw &&
|
|
69
69
|
latestComparison.raw.passed_cases === latestComparison.raw.total_cases;
|
|
70
|
+
const compileOption = {
|
|
71
|
+
value: "compile",
|
|
72
|
+
label: built ? "Rebuild portable context agents can use" : "Build portable context agents can use (Recommended)",
|
|
73
|
+
hint: built
|
|
74
|
+
? "Refresh it from the current source files"
|
|
75
|
+
: dataset.checks.length > 0
|
|
76
|
+
? latestRawPasses
|
|
77
|
+
? "Build portable context agents can use for a side-by-side comparison with the latest saved source-files run"
|
|
78
|
+
: hasSavedRawBaseline
|
|
79
|
+
? "Some questions still fail on the source files. Build portable context agents can use and compare it on the same questions"
|
|
80
|
+
: "Run the selected workflow and write portable context agents can use"
|
|
81
|
+
: "Build it for agents first, then measure it",
|
|
82
|
+
};
|
|
70
83
|
if (dataset.checks.length > 0) {
|
|
71
|
-
|
|
84
|
+
const testOption = {
|
|
72
85
|
value: "test",
|
|
73
86
|
label: built
|
|
74
|
-
? "Compare source files and
|
|
87
|
+
? "Compare source files and portable context (Recommended)"
|
|
75
88
|
: hasSavedRawBaseline
|
|
76
89
|
? "Rerun the source-files baseline"
|
|
77
|
-
: "Measure the source-files baseline
|
|
90
|
+
: "Measure the source-files baseline",
|
|
78
91
|
hint: built
|
|
79
92
|
? "See whether the portable context helps your agents on the saved questions"
|
|
80
93
|
: hasSavedRawBaseline
|
|
81
94
|
? "Refresh the saved source-files baseline on the current questions"
|
|
82
|
-
: "
|
|
83
|
-
}
|
|
95
|
+
: "Optional benchmark before Interf builds portable context",
|
|
96
|
+
};
|
|
97
|
+
if (built) {
|
|
98
|
+
options.push(testOption, compileOption);
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
options.push(compileOption, testOption);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
options.push(compileOption);
|
|
84
106
|
}
|
|
85
107
|
options.push({
|
|
86
|
-
value: "compile",
|
|
87
|
-
label: built ? "Rebuild portable context for your agents" : "Build portable context for your agents",
|
|
88
|
-
hint: built
|
|
89
|
-
? "Refresh it from the current source files"
|
|
90
|
-
: dataset.checks.length > 0
|
|
91
|
-
? latestRawPasses
|
|
92
|
-
? "Optional: build portable context for your agents for a side-by-side comparison with the latest saved source-files run"
|
|
93
|
-
: hasSavedRawBaseline
|
|
94
|
-
? "Some questions still fail on the source files. Build portable context for your agents and compare it on the same questions"
|
|
95
|
-
: "Build portable context for your agents and compare it on the same questions"
|
|
96
|
-
: "Build it for your agents first, then measure it",
|
|
97
|
-
}, {
|
|
98
108
|
value: "dataset",
|
|
99
109
|
label: "Add another setup",
|
|
100
110
|
hint: "Add a separate folder, focus, or question set",
|
|
@@ -127,14 +137,14 @@ function printDatasetRecommendation(dataset, built, latestComparison) {
|
|
|
127
137
|
if (!built) {
|
|
128
138
|
if (latestComparison?.raw) {
|
|
129
139
|
p.log.info(latestComparison.raw.passed_cases === latestComparison.raw.total_cases
|
|
130
|
-
? `Latest source-files run: ${latestComparison.raw.passed_cases}/${latestComparison.raw.total_cases}.
|
|
131
|
-
: `Latest source-files run: ${latestComparison.raw.passed_cases}/${latestComparison.raw.total_cases}. Some questions still fail on the source files. Build portable context
|
|
140
|
+
? `Latest source-files run: ${latestComparison.raw.passed_cases}/${latestComparison.raw.total_cases}. Build portable context agents can use if you want a side-by-side comparison.`
|
|
141
|
+
: `Latest source-files run: ${latestComparison.raw.passed_cases}/${latestComparison.raw.total_cases}. Some questions still fail on the source files. Build portable context agents can use if you want Interf to process this data and compare it on the same questions.`);
|
|
132
142
|
return;
|
|
133
143
|
}
|
|
134
|
-
p.log.info("Recommended first step:
|
|
144
|
+
p.log.info("Recommended first step: choose a workflow and build portable context agents can use.");
|
|
135
145
|
return;
|
|
136
146
|
}
|
|
137
|
-
p.log.info("Recommended first step: compare the source-files baseline and
|
|
147
|
+
p.log.info("Recommended first step: compare the source-files baseline and portable context.");
|
|
138
148
|
}
|
|
139
149
|
async function promptPostBaselineAction(rows) {
|
|
140
150
|
const status = currentRunTargetStatus(rows, "raw");
|
|
@@ -155,13 +165,13 @@ async function promptPostBaselineAction(rows) {
|
|
|
155
165
|
{
|
|
156
166
|
value: "compile",
|
|
157
167
|
label: compileRecommended
|
|
158
|
-
? "Build portable context
|
|
159
|
-
: "Build portable context
|
|
168
|
+
? "Build portable context agents can use (Recommended)"
|
|
169
|
+
: "Build portable context agents can use",
|
|
160
170
|
hint: compileRecommended
|
|
161
171
|
? status === "mixed"
|
|
162
|
-
? "Some selected agents still failed on the source files. Build portable context
|
|
163
|
-
: "Build portable context
|
|
164
|
-
: "Build portable context
|
|
172
|
+
? "Some selected agents still failed on the source files. Build portable context agents can use and compare it on the same questions"
|
|
173
|
+
: "Build portable context agents can use and compare it on the same questions"
|
|
174
|
+
: "Build portable context agents can use and compare it on the same questions",
|
|
165
175
|
},
|
|
166
176
|
{
|
|
167
177
|
value: "edit",
|
|
@@ -220,29 +230,21 @@ async function chooseCompiledForWizard(options) {
|
|
|
220
230
|
return findSavedCompiledConfig(options.sourcePath, String(selected));
|
|
221
231
|
}
|
|
222
232
|
async function promptCompiledSetup(options) {
|
|
223
|
-
let workflowId = options.initial?.workflow ?? "interf-default";
|
|
224
|
-
let workflowLabel = buildCompiledWorkflowOptions(options.sourcePath)
|
|
225
|
-
.find((option) => option.value === workflowId)?.label ?? workflowId;
|
|
226
|
-
if (options.introStyle === "edit") {
|
|
227
|
-
const workflowChoice = await chooseCompiledWorkflow(options.sourcePath, {
|
|
228
|
-
currentWorkflowId: workflowId,
|
|
229
|
-
message: "Which workflow should this dataset use?",
|
|
230
|
-
});
|
|
231
|
-
if (p.isCancel(workflowChoice))
|
|
232
|
-
return null;
|
|
233
|
-
workflowId = workflowChoice;
|
|
234
|
-
workflowLabel = buildCompiledWorkflowOptions(options.sourcePath)
|
|
235
|
-
.find((option) => option.value === workflowId)?.label ?? workflowId;
|
|
236
|
-
}
|
|
237
233
|
const compiledConfig = await promptSingleCompiledConfig({
|
|
238
234
|
projectPath: options.sourcePath,
|
|
239
235
|
initial: options.initial,
|
|
240
236
|
...(options.fixedName ? { fixedName: options.fixedName } : {}),
|
|
241
237
|
introStyle: options.introStyle,
|
|
242
|
-
selectedWorkflowLabel: workflowLabel,
|
|
243
238
|
});
|
|
244
239
|
if (!compiledConfig)
|
|
245
240
|
return null;
|
|
241
|
+
const workflowChoice = await chooseOrCreateCompiledWorkflowForDataset(options.sourcePath, compiledConfig, {
|
|
242
|
+
currentWorkflowId: options.initial?.workflow ?? "interf-default",
|
|
243
|
+
executionProfile: options.executionProfile,
|
|
244
|
+
});
|
|
245
|
+
if (!workflowChoice || p.isCancel(workflowChoice))
|
|
246
|
+
return null;
|
|
247
|
+
const workflowId = String(workflowChoice);
|
|
246
248
|
const compiledConfigWithWorkflow = {
|
|
247
249
|
...compiledConfig,
|
|
248
250
|
workflow: workflowId,
|
|
@@ -254,12 +256,15 @@ async function promptCompiledSetup(options) {
|
|
|
254
256
|
if (builtCompiledPath) {
|
|
255
257
|
syncCompiledInterfConfigFromSourceDatasetConfig(builtCompiledPath, compiledConfigWithWorkflow);
|
|
256
258
|
}
|
|
259
|
+
else {
|
|
260
|
+
ensurePortableContextScaffold(options.sourcePath, compiledConfigWithWorkflow.name, compiledConfigWithWorkflow.workflow ?? "interf-default");
|
|
261
|
+
}
|
|
257
262
|
console.log();
|
|
258
|
-
console.log(chalk.green(` ✓ Saved setup in ${
|
|
263
|
+
console.log(chalk.green(` ✓ Saved setup in ${SOURCE_FOLDER_CONFIG_PATH}`));
|
|
259
264
|
console.log(chalk.dim(` Project folder: ${options.sourcePath}`));
|
|
260
265
|
console.log(chalk.dim(` Setup: ${compiledConfigWithWorkflow.name}`));
|
|
261
266
|
console.log(chalk.dim(` Source folder: ${compiledConfigWithWorkflow.path}`));
|
|
262
|
-
console.log(chalk.dim(` Workflow: ${
|
|
267
|
+
console.log(chalk.dim(` Workflow: ${workflowId} · interf/workflows/${workflowId}/`));
|
|
263
268
|
console.log(chalk.dim(` Compile mode: ${describeCompileLoopSelection({
|
|
264
269
|
maxAttempts: compiledConfigWithWorkflow.max_attempts,
|
|
265
270
|
maxLoops: compiledConfigWithWorkflow.max_loops,
|
|
@@ -318,6 +323,9 @@ async function runCompiledActionMenu(sourcePath, compiledConfig, options = {}) {
|
|
|
318
323
|
if (builtCompiledPath) {
|
|
319
324
|
syncCompiledInterfConfigFromSourceDatasetConfig(builtCompiledPath, nextConfig);
|
|
320
325
|
}
|
|
326
|
+
else {
|
|
327
|
+
ensurePortableContextScaffold(sourcePath, nextConfig.name, nextConfig.workflow ?? "interf-default");
|
|
328
|
+
}
|
|
321
329
|
p.log.info(`Assigned workflow "${workflowId}" to dataset "${compiledConfig.name}".`);
|
|
322
330
|
p.log.info("Next: run `interf compile`, then `interf test`.");
|
|
323
331
|
}
|
|
@@ -408,7 +416,7 @@ async function runCompiledActionMenu(sourcePath, compiledConfig, options = {}) {
|
|
|
408
416
|
return;
|
|
409
417
|
}
|
|
410
418
|
if (compileResult.testedDuringCompile) {
|
|
411
|
-
p.log.info("This compile run already checked the portable context
|
|
419
|
+
p.log.info("This compile run already checked the portable context on the saved questions.");
|
|
412
420
|
p.log.info("Run `interf test` later if you want a fresh side-by-side comparison summary.");
|
|
413
421
|
return;
|
|
414
422
|
}
|
|
@@ -427,14 +435,14 @@ async function runCompiledActionMenu(sourcePath, compiledConfig, options = {}) {
|
|
|
427
435
|
if (compiledConfig.checks.length === 0)
|
|
428
436
|
return;
|
|
429
437
|
if (compileResult.testedDuringCompile) {
|
|
430
|
-
p.log.info("This compile run already checked the portable context
|
|
438
|
+
p.log.info("This compile run already checked the portable context on the compile agent.");
|
|
431
439
|
}
|
|
432
440
|
const runCompiledTest = await p.confirm({
|
|
433
441
|
message: compileResult.testedDuringCompile
|
|
434
442
|
? "Run a fresh source-files versus portable-context comparison now?"
|
|
435
443
|
: builtCompiledPath
|
|
436
|
-
? "Run source files and
|
|
437
|
-
: "Compare source files and
|
|
444
|
+
? "Run source files and portable context on the saved questions now?"
|
|
445
|
+
: "Compare source files and portable context on the saved questions now?",
|
|
438
446
|
initialValue: true,
|
|
439
447
|
});
|
|
440
448
|
if (p.isCancel(runCompiledTest) || !runCompiledTest)
|
|
@@ -457,17 +465,20 @@ export const initCommand = {
|
|
|
457
465
|
};
|
|
458
466
|
export async function runInitCommand() {
|
|
459
467
|
p.intro(chalk.bold("Interf"));
|
|
460
|
-
p.log.info("Interf prepares
|
|
461
|
-
p.log.info("
|
|
468
|
+
p.log.info("Interf prepares data for agent work.");
|
|
469
|
+
p.log.info("Run locally, process files, show evidence that your data is ready, and write verifiable outputs as portable context for agents.");
|
|
462
470
|
const cwd = process.cwd();
|
|
463
471
|
const detected = detectInterf(cwd);
|
|
464
472
|
const sourcePath = detected ? resolveSourceControlPath(detected.path) : cwd;
|
|
465
473
|
if (detected) {
|
|
466
474
|
p.log.info(`Working from the project folder: ${sourcePath}`);
|
|
467
475
|
}
|
|
476
|
+
if (migrateLegacySourceFolderConfig(sourcePath)) {
|
|
477
|
+
p.log.info(`Moved saved setup to ${SOURCE_FOLDER_CONFIG_PATH}.`);
|
|
478
|
+
}
|
|
468
479
|
const seeded = seedLocalDefaultWorkflow({ sourcePath });
|
|
469
480
|
if (!seeded.alreadyExisted) {
|
|
470
|
-
p.log.info(`Copied built-in
|
|
481
|
+
p.log.info(`Copied built-in workflow package to interf/workflows/${seeded.workflowId}/ — inspect or fork it anytime.`);
|
|
471
482
|
}
|
|
472
483
|
const savedEntries = listSavedCompiledEntries(sourcePath);
|
|
473
484
|
if (savedEntries.length === 0) {
|
|
@@ -544,7 +544,7 @@ export async function promptCompileLoopSelection(options) {
|
|
|
544
544
|
once: {
|
|
545
545
|
value: "once",
|
|
546
546
|
label: "Compile once",
|
|
547
|
-
hint: "Build portable context
|
|
547
|
+
hint: "Build portable context agents can use once with the selected workflow",
|
|
548
548
|
},
|
|
549
549
|
"self-improving": {
|
|
550
550
|
value: "self-improving",
|
|
@@ -651,9 +651,6 @@ export async function promptSingleCompiledConfig(options = {}) {
|
|
|
651
651
|
p.log.info(`Dataset name: ${name}`);
|
|
652
652
|
}
|
|
653
653
|
}
|
|
654
|
-
if (options.selectedWorkflowLabel) {
|
|
655
|
-
p.log.info(`Workflow: ${options.selectedWorkflowLabel}`);
|
|
656
|
-
}
|
|
657
654
|
let compileLoopSelection = {
|
|
658
655
|
...(typeof initial?.max_attempts === "number" ? { max_attempts: initial.max_attempts } : {}),
|
|
659
656
|
...(typeof initial?.max_loops === "number" ? { max_loops: initial.max_loops } : {}),
|
|
@@ -18,7 +18,7 @@ function statusColor(status) {
|
|
|
18
18
|
}
|
|
19
19
|
export const statusCommand = {
|
|
20
20
|
command: "status",
|
|
21
|
-
describe: "Show deterministic health for the portable context
|
|
21
|
+
describe: "Show deterministic health for the portable context agents use",
|
|
22
22
|
handler: async () => {
|
|
23
23
|
let compiledPath = null;
|
|
24
24
|
const detected = detectInterf(process.cwd());
|
|
@@ -34,7 +34,7 @@ export const statusCommand = {
|
|
|
34
34
|
if (local.length === 0) {
|
|
35
35
|
process.exitCode = 1;
|
|
36
36
|
console.log(chalk.red(" No portable contexts found."));
|
|
37
|
-
console.log(chalk.dim(" Run `interf`, save questions, and compile portable context
|
|
37
|
+
console.log(chalk.dim(" Run `interf`, save questions, and compile portable context agents can use first."));
|
|
38
38
|
return;
|
|
39
39
|
}
|
|
40
40
|
if (local.length === 1) {
|
|
@@ -75,7 +75,7 @@ export const statusCommand = {
|
|
|
75
75
|
console.log();
|
|
76
76
|
console.log(chalk.dim(latestComparisonState.stale
|
|
77
77
|
? " Saved test results are stale for the current questions. Run `interf test` again."
|
|
78
|
-
: " No saved comparison yet. Run `interf test` to measure source files and
|
|
78
|
+
: " No saved comparison yet. Run `interf test` to measure source files and portable context."));
|
|
79
79
|
}
|
|
80
80
|
console.log();
|
|
81
81
|
const metricOrder = [
|
|
@@ -3,7 +3,8 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
4
|
import { createRawTestTarget, createCompiledTestTarget, runTargetTestsAuto, saveTargetTestRun, } from "../../packages/testing/test.js";
|
|
5
5
|
import { buildTestSpecFromSourceFolderConfig, buildTestSpecFromCompiledDatasetConfig, fingerprintTruthChecks, resolveSourceDatasetPath, } from "../../packages/project-model/source-config.js";
|
|
6
|
-
import {
|
|
6
|
+
import { ensurePortableContextScaffold, } from "../../packages/project-model/interf.js";
|
|
7
|
+
import { datasetLatestTestStatePath, datasetLatestTestSummaryPath, normalizeDatasetTestRunId, datasetTestRunPath, datasetTestRunsRoot, } from "../../packages/project-model/project-paths.js";
|
|
7
8
|
import { testRootForCompiled } from "../../packages/project-model/compiled-paths.js";
|
|
8
9
|
import { readJsonFileWithSchema } from "../../packages/shared/parse.js";
|
|
9
10
|
import { TestRunComparisonSchema } from "../../packages/testing/lib/schema.js";
|
|
@@ -137,7 +138,7 @@ export function printAgentTestMatrix(rows) {
|
|
|
137
138
|
const headers = [
|
|
138
139
|
"Agent",
|
|
139
140
|
...(includeRaw ? ["Source files"] : []),
|
|
140
|
-
...(includeCompiled ? ["
|
|
141
|
+
...(includeCompiled ? ["Portable context"] : []),
|
|
141
142
|
...(includeDelta ? ["Delta"] : []),
|
|
142
143
|
];
|
|
143
144
|
const body = rows.map((row) => [
|
|
@@ -162,7 +163,7 @@ export function printAgentTestFailures(rows) {
|
|
|
162
163
|
const failures = [];
|
|
163
164
|
for (const [label, outcome] of [
|
|
164
165
|
["Source files", row.rawOutcome ?? null],
|
|
165
|
-
["
|
|
166
|
+
["Portable context", row.compiledOutcome ?? null],
|
|
166
167
|
]) {
|
|
167
168
|
if (!outcome || outcome.result.ok)
|
|
168
169
|
continue;
|
|
@@ -232,7 +233,7 @@ export function printSavedTestComparison(rawOutcome, compiledOutcome, comparison
|
|
|
232
233
|
if (compiledOutcome) {
|
|
233
234
|
if (rawOutcome)
|
|
234
235
|
console.log();
|
|
235
|
-
printSavedTestOutcome("
|
|
236
|
+
printSavedTestOutcome("Portable context", compiledOutcome);
|
|
236
237
|
}
|
|
237
238
|
if (rawOutcome && compiledOutcome) {
|
|
238
239
|
const rawQuestions = questionPassRate(rawOutcome);
|
|
@@ -260,7 +261,7 @@ export function printSavedTestComparisonSummary(rawOutcome, compiledOutcome, com
|
|
|
260
261
|
console.log(` | Source files | \`${rawOutcome.result.passedCases}/${rawOutcome.result.totalCases}\` |`);
|
|
261
262
|
}
|
|
262
263
|
if (compiledOutcome) {
|
|
263
|
-
console.log(` |
|
|
264
|
+
console.log(` | Portable context | \`${compiledOutcome.result.passedCases}/${compiledOutcome.result.totalCases}\` |`);
|
|
264
265
|
}
|
|
265
266
|
if (rawOutcome && compiledOutcome) {
|
|
266
267
|
const rawQuestions = questionPassRate(rawOutcome);
|
|
@@ -282,7 +283,7 @@ export function saveTestComparisonRun(options) {
|
|
|
282
283
|
? summarizeSavedTestOutcome("Source files", options.rawOutcome)
|
|
283
284
|
: null;
|
|
284
285
|
const compiledSummary = options.compiledOutcome
|
|
285
|
-
? summarizeSavedTestOutcome("
|
|
286
|
+
? summarizeSavedTestOutcome("Portable context", options.compiledOutcome)
|
|
286
287
|
: null;
|
|
287
288
|
const effectiveMode = rawSummary && compiledSummary
|
|
288
289
|
? "both"
|
|
@@ -343,11 +344,12 @@ export async function runSavedRawTest(options) {
|
|
|
343
344
|
return null;
|
|
344
345
|
}
|
|
345
346
|
const datasetSourcePath = resolveSourceDatasetPath(options.sourcePath, options.datasetConfig);
|
|
347
|
+
const portableContextPath = ensurePortableContextScaffold(options.sourcePath, options.datasetConfig.name, options.datasetConfig.workflow ?? "interf");
|
|
346
348
|
const target = createRawTestTarget(datasetSourcePath);
|
|
347
349
|
const run = await runTargetTestsAuto(datasetSourcePath, spec, [target], {
|
|
348
350
|
executor,
|
|
349
351
|
preserveSandboxes: options.preserveSandboxes ?? "on-failure",
|
|
350
|
-
artifactRootPath:
|
|
352
|
+
artifactRootPath: testRootForCompiled(portableContextPath),
|
|
351
353
|
});
|
|
352
354
|
const result = run.results[0];
|
|
353
355
|
if (!result)
|
|
@@ -9,7 +9,7 @@ import { printAgentTestFailures, printAgentTestMatrix, printSavedTestComparisonS
|
|
|
9
9
|
import { listRunAgentOptions, promptForTestAgents, resolveNamedLocalExecutor, resolveOrConfigureLocalExecutor, } from "./executor-flow.js";
|
|
10
10
|
export const testCommand = {
|
|
11
11
|
command: "test",
|
|
12
|
-
describe: "Compare source files and
|
|
12
|
+
describe: "Compare source files and portable context on saved questions",
|
|
13
13
|
builder: (yargs) => addExecutionProfileOptions(yargs)
|
|
14
14
|
.option("dataset", {
|
|
15
15
|
type: "string",
|
|
@@ -17,7 +17,7 @@ export const testCommand = {
|
|
|
17
17
|
})
|
|
18
18
|
.option("target", {
|
|
19
19
|
choices: ["both", "raw", "compiled"],
|
|
20
|
-
describe: "Test source files,
|
|
20
|
+
describe: "Test source files, portable context, or both. Default: both when portable context exists, otherwise source files.",
|
|
21
21
|
})
|
|
22
22
|
.option("keep-sandboxes", {
|
|
23
23
|
type: "boolean",
|
|
@@ -49,7 +49,7 @@ async function promptTestMode(hasBuiltCompiled) {
|
|
|
49
49
|
options: [
|
|
50
50
|
{
|
|
51
51
|
value: "both",
|
|
52
|
-
label: "Compare source files and
|
|
52
|
+
label: "Compare source files and portable context (Recommended)",
|
|
53
53
|
hint: "Measure whether the portable context helps your agents on the saved questions",
|
|
54
54
|
},
|
|
55
55
|
{
|
|
@@ -60,7 +60,7 @@ async function promptTestMode(hasBuiltCompiled) {
|
|
|
60
60
|
{
|
|
61
61
|
value: "compiled",
|
|
62
62
|
label: "Portable context only",
|
|
63
|
-
hint: "Measure the current portable context
|
|
63
|
+
hint: "Measure the current portable context on the saved questions",
|
|
64
64
|
},
|
|
65
65
|
],
|
|
66
66
|
});
|
|
@@ -240,13 +240,13 @@ export async function runTestCommand(argv = {}) {
|
|
|
240
240
|
return false;
|
|
241
241
|
if (selectedMode === "compiled" && !hasBuiltCompiled) {
|
|
242
242
|
process.exitCode = 1;
|
|
243
|
-
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context
|
|
243
|
+
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context agents can use yet.`));
|
|
244
244
|
console.log(chalk.dim(" Run `interf compile` first."));
|
|
245
245
|
return false;
|
|
246
246
|
}
|
|
247
247
|
if (selectedMode === "both" && !hasBuiltCompiled) {
|
|
248
248
|
process.exitCode = 1;
|
|
249
|
-
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context
|
|
249
|
+
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context agents can use yet, so Interf cannot compare source files and portable context.`));
|
|
250
250
|
console.log(chalk.dim(" Run `interf compile` first, or rerun with `--target raw`."));
|
|
251
251
|
return false;
|
|
252
252
|
}
|
|
@@ -268,7 +268,7 @@ export async function runTestCommand(argv = {}) {
|
|
|
268
268
|
})));
|
|
269
269
|
if ((mode === "compiled" || mode === "both") && rows.some((row) => !row.compiledOutcome)) {
|
|
270
270
|
process.exitCode = 1;
|
|
271
|
-
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context
|
|
271
|
+
console.log(chalk.red(` Setup "${selectedCompiled.name}" does not have portable context agents can use yet.`));
|
|
272
272
|
console.log(chalk.dim(" Run `interf compile` first."));
|
|
273
273
|
return false;
|
|
274
274
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export { createCompiled, } from "./packages/project-model/interf.js";
|
|
|
8
8
|
export { compileCompiled, runCompiledSummarize, runCompiledCompile, } from "./packages/compiler/workflows.js";
|
|
9
9
|
export { createRawTestTarget, createCompiledTestTarget, listTestSpecs, loadTestSpec, loadTestSpecFromFile, writeTestSpec, listTestTargets, runTargetTests, runTargetTestsWithJudge, runTargetTestsAuto, saveTargetTestRun, } from "./packages/testing/test.js";
|
|
10
10
|
export { computeCompiledHealth, } from "./packages/compiler/state.js";
|
|
11
|
-
export { SOURCE_FOLDER_CONFIG_FILE, loadSourceFolderConfig, buildTestSpecFromSourceFolderConfig, } from "./packages/project-model/source-config.js";
|
|
11
|
+
export { SOURCE_FOLDER_CONFIG_FILE, SOURCE_FOLDER_CONFIG_PATH, legacySourceFolderConfigPath, loadSourceFolderConfig, migrateLegacySourceFolderConfig, resolveSourceFolderConfigPath, sourceFolderConfigPath, buildTestSpecFromSourceFolderConfig, } from "./packages/project-model/source-config.js";
|
|
12
12
|
export { validateCompiledSummarize, validateCompiledCompile, } from "./packages/compiler/validate.js";
|
|
13
13
|
export { InterfConfigSchema, SourceFolderConfigSchema, } from "./packages/project-model/lib/schema.js";
|
|
14
14
|
export { TestSpecSchema, } from "./packages/testing/lib/schema.js";
|
package/dist/index.js
CHANGED
|
@@ -8,7 +8,7 @@ export { createCompiled, } from "./packages/project-model/interf.js";
|
|
|
8
8
|
export { compileCompiled, runCompiledSummarize, runCompiledCompile, } from "./packages/compiler/workflows.js";
|
|
9
9
|
export { createRawTestTarget, createCompiledTestTarget, listTestSpecs, loadTestSpec, loadTestSpecFromFile, writeTestSpec, listTestTargets, runTargetTests, runTargetTestsWithJudge, runTargetTestsAuto, saveTargetTestRun, } from "./packages/testing/test.js";
|
|
10
10
|
export { computeCompiledHealth, } from "./packages/compiler/state.js";
|
|
11
|
-
export { SOURCE_FOLDER_CONFIG_FILE, loadSourceFolderConfig, buildTestSpecFromSourceFolderConfig, } from "./packages/project-model/source-config.js";
|
|
11
|
+
export { SOURCE_FOLDER_CONFIG_FILE, SOURCE_FOLDER_CONFIG_PATH, legacySourceFolderConfigPath, loadSourceFolderConfig, migrateLegacySourceFolderConfig, resolveSourceFolderConfigPath, sourceFolderConfigPath, buildTestSpecFromSourceFolderConfig, } from "./packages/project-model/source-config.js";
|
|
12
12
|
export { validateCompiledSummarize, validateCompiledCompile, } from "./packages/compiler/validate.js";
|
|
13
13
|
export { InterfConfigSchema, SourceFolderConfigSchema, } from "./packages/project-model/lib/schema.js";
|
|
14
14
|
export { TestSpecSchema, } from "./packages/testing/lib/schema.js";
|
|
@@ -42,7 +42,7 @@ export function renderCompiledAgents(compiledPath, name, workflowId, about, opti
|
|
|
42
42
|
`# ${name}`,
|
|
43
43
|
"",
|
|
44
44
|
"This is portable context built by Interf.",
|
|
45
|
-
"
|
|
45
|
+
"This folder gives agents prepared evidence, structure, and cross-file connections so they do not have to rediscover the full picture from source files during the job.",
|
|
46
46
|
"",
|
|
47
47
|
...(about
|
|
48
48
|
? [
|
|
@@ -56,17 +56,17 @@ export function renderCompiledAgents(compiledPath, name, workflowId, about, opti
|
|
|
56
56
|
"2. Let the workflow docs and declared output zones guide retrieval instead of assuming a fixed note layout.",
|
|
57
57
|
"3. Use `raw/` when you need direct quotes, exact chart values, table lookups, or verification.",
|
|
58
58
|
"",
|
|
59
|
-
"## How this
|
|
59
|
+
"## How this portable context works",
|
|
60
60
|
"",
|
|
61
|
-
"- The workflow package defines the context interface this
|
|
61
|
+
"- The workflow package defines the context interface this portable context implements on disk.",
|
|
62
62
|
"- `.interf/interf.json` points to the local `raw/` snapshot via `source.path` and back to the project root via `source.control_path`.",
|
|
63
63
|
`- Workflow seed: \`${workflowOriginSelected}\`.`,
|
|
64
64
|
...(workflowLocalDraft
|
|
65
|
-
? ["- This
|
|
65
|
+
? ["- This portable context now carries a local workflow draft improved from that seed. Recompiling this portable context reuses the local `.interf/workflow/` package."]
|
|
66
66
|
: []),
|
|
67
67
|
`- Active local workflow id: \`${workflowId}\`.`,
|
|
68
|
-
"- `.interf/workflow/` is the local editable method package for this
|
|
69
|
-
`- \`.interf/workflow/${WORKFLOW_SCHEMA_FILE}\` is the deterministic context interface for this
|
|
68
|
+
"- `.interf/workflow/` is the local editable method package for this portable context.",
|
|
69
|
+
`- \`.interf/workflow/${WORKFLOW_SCHEMA_FILE}\` is the deterministic context interface for this portable context.`,
|
|
70
70
|
"- `.interf/workflow/improve/` is the editable source for workflow-improvement loops.",
|
|
71
71
|
"- `.interf/workflow/use/query/` is the editable source for the generated native query shell.",
|
|
72
72
|
"- `.interf/workflow/compile/stages/` defines stage-specific docs that Interf projects into native execution shells for automated runs.",
|
|
@@ -83,7 +83,7 @@ export function renderCompiledAgents(compiledPath, name, workflowId, about, opti
|
|
|
83
83
|
"- Prefer the workflow-declared context outputs before `raw/`.",
|
|
84
84
|
"- Use the generated native `interf-query` skill for manual querying. The editable source lives at `.interf/workflow/use/query/SKILL.md`.",
|
|
85
85
|
"- Treat `.interf/` as method/runtime metadata, not answer evidence, unless explicitly asked to inspect workflow or test history.",
|
|
86
|
-
"- Use `raw/` for quotes, verification, ambiguity, or evidence the
|
|
86
|
+
"- Use `raw/` for quotes, verification, ambiguity, or evidence the portable context does not expose well.",
|
|
87
87
|
"- If exact chart, table, or image-derived evidence matters, inspect the raw source and say whether the answer was text-derived, table-derived, or chart-derived.",
|
|
88
88
|
"",
|
|
89
89
|
"## Commands",
|
|
@@ -92,14 +92,14 @@ export function renderCompiledAgents(compiledPath, name, workflowId, about, opti
|
|
|
92
92
|
"interf compile build this portable context",
|
|
93
93
|
"interf test run checks against this portable context",
|
|
94
94
|
"interf verify stage <id> verify one workflow stage",
|
|
95
|
-
"interf verify compiled verify the full
|
|
95
|
+
"interf verify compiled verify the full portable context output",
|
|
96
96
|
"interf status show deterministic health",
|
|
97
97
|
"```",
|
|
98
98
|
"",
|
|
99
99
|
"## Rules",
|
|
100
100
|
"",
|
|
101
101
|
"- Do not modify files under `raw/`.",
|
|
102
|
-
"- Treat
|
|
102
|
+
"- Treat prepared notes as working context, not final truth.",
|
|
103
103
|
"- When confidence is low, verify against `raw/` before answering strongly.",
|
|
104
104
|
"",
|
|
105
105
|
].join("\n");
|
|
@@ -113,15 +113,15 @@ export function renderCompiledQuerySkill() {
|
|
|
113
113
|
"Default loop:",
|
|
114
114
|
"1. Read `workflow/README.md` and this file first.",
|
|
115
115
|
`2. Use the workflow zones declared in \`workflow/${WORKFLOW_SCHEMA_FILE}\` before falling back to \`raw/\`.`,
|
|
116
|
-
"3. Use `raw/` for direct quotes, verification, exact lookups, and cases where the
|
|
116
|
+
"3. Use `raw/` for direct quotes, verification, exact lookups, and cases where the portable context is missing the needed evidence or is ambiguous.",
|
|
117
117
|
"",
|
|
118
118
|
"Answering rule:",
|
|
119
119
|
"- do not modify files under `raw/`",
|
|
120
|
-
"- treat the workflow as the method layer and the
|
|
120
|
+
"- treat the workflow as the method layer and the portable-context zones as the working retrieval surface",
|
|
121
121
|
"- say explicitly when an answer depends on approximation, bounded inference, or a raw-source re-check",
|
|
122
|
-
"- use `raw/` to confirm source page, metric family, provenance, or exact wording when the
|
|
122
|
+
"- use `raw/` to confirm source page, metric family, provenance, or exact wording when the portable context is missing the needed evidence or is ambiguous",
|
|
123
123
|
"- do not invent navigation or note structure beyond what this workflow declares",
|
|
124
|
-
"- when the
|
|
124
|
+
"- when the portable context is insufficient, verify in `raw/` and then answer",
|
|
125
125
|
"",
|
|
126
126
|
"You can edit this file to bias manual question-answering behavior for this portable context.",
|
|
127
127
|
"",
|
|
@@ -398,14 +398,13 @@ function renderCompiledQueryNativeSkill(querySkillContent) {
|
|
|
398
398
|
"---",
|
|
399
399
|
"name: interf-query",
|
|
400
400
|
"description: >",
|
|
401
|
-
" Native local query skill for this portable context built by Interf",
|
|
402
|
-
"
|
|
403
|
-
" and its raw fallback.",
|
|
401
|
+
" Native local query skill for this portable context built by Interf.",
|
|
402
|
+
" Use it for manual questions against the portable context and raw fallback.",
|
|
404
403
|
"---",
|
|
405
404
|
"",
|
|
406
405
|
"# Interf Query",
|
|
407
406
|
"",
|
|
408
|
-
"This is the native local query skill for
|
|
407
|
+
"This is the native local query skill for portable context built by Interf.",
|
|
409
408
|
"Use it when reading this portable context manually.",
|
|
410
409
|
"Editable source: `.interf/workflow/use/query/SKILL.md`.",
|
|
411
410
|
"",
|
|
@@ -667,7 +666,7 @@ function renderWorkflowAuthoringAgents(options) {
|
|
|
667
666
|
`# ${options.label} — Workflow Authoring Shell`,
|
|
668
667
|
"",
|
|
669
668
|
"This is an automated workflow-authoring shell generated by Interf.",
|
|
670
|
-
"It exists to create or refine one standalone workflow
|
|
669
|
+
"It exists to create or refine one standalone workflow from the source data, desired portable-context outputs, and proof requirements in this task.",
|
|
671
670
|
"",
|
|
672
671
|
"## Start Here",
|
|
673
672
|
"",
|
|
@@ -683,12 +682,14 @@ function renderWorkflowAuthoringAgents(options) {
|
|
|
683
682
|
"- Edit only files under `workflow/`.",
|
|
684
683
|
"- Do not edit raw files or preview artifacts.",
|
|
685
684
|
"- Keep the workflow valid for the current compiler API and workflow schema.",
|
|
685
|
+
"- For acceptance rules, use `zone_counts_at_least` for numeric fixed minimums and `zone_counts_at_least_counts` only for runtime count-key strings such as `source_total`.",
|
|
686
686
|
"- Prefer direct file-reading and search tools over shell commands for routine file inspection.",
|
|
687
687
|
"- Do not use shell helpers like `cat`, `sed`, `ls`, or `find` when a native read/search tool can inspect the same files.",
|
|
688
688
|
"",
|
|
689
689
|
"## Goal",
|
|
690
690
|
"",
|
|
691
|
-
"- produce one standalone workflow tuned to the
|
|
691
|
+
"- produce one standalone workflow tuned to the source data, desired outputs, and checks in this task",
|
|
692
|
+
"- define output zones and stage proof so Interf can show whether the data is ready",
|
|
692
693
|
"- preserve deterministic stage and context-interface contracts",
|
|
693
694
|
"- stop once the workflow edits are complete",
|
|
694
695
|
"",
|
|
@@ -710,6 +711,7 @@ function renderWorkflowAuthoringSkill() {
|
|
|
710
711
|
"Rules:",
|
|
711
712
|
"- edit only `workflow/`",
|
|
712
713
|
`- keep \`workflow.json\`, \`${WORKFLOW_SCHEMA_FILE}\`, and any changed stage docs aligned`,
|
|
714
|
+
"- use `zone_counts_at_least` for numeric fixed minimums; use `zone_counts_at_least_counts` only for runtime count-key strings such as `source_total`",
|
|
713
715
|
"- prefer small, defensible package edits over random churn",
|
|
714
716
|
"- keep the package standalone; do not introduce runtime inheritance or fallback assumptions",
|
|
715
717
|
"- do not introduce wikilinks unless the workflow also creates the target note by exact basename or explicit relative path",
|
|
@@ -852,6 +854,7 @@ function renderWorkflowImprovementAgents(compiledName, workflowId, loopIndex) {
|
|
|
852
854
|
"- Do not edit checks, test specs, raw files, or generated context outputs.",
|
|
853
855
|
"- Review context outputs under `artifacts/compiled-view/` and test/runtime evidence under `artifacts/`.",
|
|
854
856
|
"- Keep the workflow valid for the current compiler API and workflow schema.",
|
|
857
|
+
"- For acceptance rules, use `zone_counts_at_least` for numeric fixed minimums and `zone_counts_at_least_counts` only for runtime count-key strings such as `source_total`.",
|
|
855
858
|
"",
|
|
856
859
|
"## Goal",
|
|
857
860
|
"",
|
|
@@ -98,7 +98,7 @@ export function resolveSourceControlPathForCompiled(compiledPath) {
|
|
|
98
98
|
break;
|
|
99
99
|
cursor = parent;
|
|
100
100
|
}
|
|
101
|
-
throw new Error(`
|
|
101
|
+
throw new Error(`Compiled context is not under the canonical project layout: ${compiledPath}`);
|
|
102
102
|
}
|
|
103
103
|
export function defaultControlPathForCompiled(compiledPath) {
|
|
104
104
|
const relativePath = relative(compiledPath, resolveSourceControlPathForCompiled(compiledPath));
|
|
@@ -28,7 +28,7 @@ export function buildStagePrompt(instructions, contractPath, statusLines) {
|
|
|
28
28
|
"Prefer direct file-reading and search tools over shell commands when you inspect inputs, docs, or generated outputs.",
|
|
29
29
|
"Do not use shell helpers like `cat`, `sed`, `ls`, or `find` for routine file inspection if a native read/search tool can do the same job.",
|
|
30
30
|
"If the contract lists `instructions.local_docs`, open every one of those files before you write artifacts or declare the stage complete.",
|
|
31
|
-
"If `AGENTS.md` is listed in the contract, use it only for context
|
|
31
|
+
"If `AGENTS.md` is listed in the contract, use it only for portable-context routing that is directly relevant to this stage.",
|
|
32
32
|
"`.interf/runtime/run.json`, `.interf/runtime/state.json`, `.interf/runtime/health.json`, and `.interf/runtime/view-spec.json` are runtime artifacts written by Interf. You may read them, but do not create, edit, or replace them unless the contract explicitly marks a different proof artifact as stage-owned.",
|
|
33
33
|
"If a contract-listed output file does not exist yet, create it in one whole-file write. Do not attempt patch-style edits against a missing runtime path.",
|
|
34
34
|
"Stay inside the current compiled context. Do not try to open repo docs, repo source files, or other paths outside the compiled context unless the stage contract explicitly requires them.",
|