@interf/compiler 0.6.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -80
- package/builtin-workflows/interf/README.md +5 -5
- package/builtin-workflows/interf/compile/stages/shape/SKILL.md +4 -4
- package/builtin-workflows/interf/improve/SKILL.md +2 -2
- package/builtin-workflows/interf/use/query/SKILL.md +1 -1
- package/builtin-workflows/interf/workflow.json +6 -6
- package/builtin-workflows/interf/workflow.schema.json +1 -1
- package/dist/bin.js +2 -28
- package/dist/{commands → cli/commands}/check-draft.d.ts +2 -2
- package/dist/{commands → cli/commands}/check-draft.js +16 -16
- package/dist/{commands → cli/commands}/compile-controller.d.ts +3 -3
- package/dist/{commands → cli/commands}/compile-controller.js +37 -35
- package/dist/{commands → cli/commands}/compile.d.ts +2 -2
- package/dist/{commands → cli/commands}/compile.js +15 -11
- package/dist/{commands → cli/commands}/compiled-flow.d.ts +3 -4
- package/dist/{commands → cli/commands}/compiled-flow.js +9 -14
- package/dist/{commands → cli/commands}/create-workflow-wizard.d.ts +3 -3
- package/dist/{commands → cli/commands}/create-workflow-wizard.js +64 -37
- package/dist/{commands → cli/commands}/create.d.ts +1 -1
- package/dist/{commands → cli/commands}/create.js +15 -18
- package/dist/{commands → cli/commands}/dataset-selection.d.ts +1 -1
- package/dist/{commands → cli/commands}/default.js +3 -3
- package/dist/{commands → cli/commands}/doctor.js +5 -6
- package/dist/{commands → cli/commands}/executor-flow.d.ts +1 -1
- package/dist/{commands → cli/commands}/executor-flow.js +13 -16
- package/dist/{commands → cli/commands}/init.d.ts +4 -1
- package/dist/{commands → cli/commands}/init.js +162 -77
- package/dist/{commands → cli/commands}/list.js +6 -6
- package/dist/{commands → cli/commands}/reset.js +4 -4
- package/dist/{commands → cli/commands}/source-config-wizard.d.ts +9 -4
- package/dist/{commands → cli/commands}/source-config-wizard.js +168 -94
- package/dist/{commands → cli/commands}/status.js +20 -11
- package/dist/{commands → cli/commands}/test-flow.d.ts +13 -4
- package/dist/{commands → cli/commands}/test-flow.js +52 -48
- package/dist/cli/commands/test.d.ts +14 -0
- package/dist/{commands → cli/commands}/test.js +40 -24
- package/dist/{commands → cli/commands}/verify.js +4 -4
- package/dist/cli/index.d.ts +21 -0
- package/dist/cli/index.js +33 -0
- package/dist/index.d.ts +22 -11
- package/dist/index.js +15 -6
- package/dist/lib/agent-args.d.ts +1 -4
- package/dist/lib/agent-args.js +1 -52
- package/dist/lib/agent-constants.d.ts +1 -5
- package/dist/lib/agent-constants.js +1 -28
- package/dist/lib/agent-detection.d.ts +1 -7
- package/dist/lib/agent-detection.js +1 -65
- package/dist/lib/agent-execution.d.ts +1 -2
- package/dist/lib/agent-execution.js +1 -243
- package/dist/lib/agent-logs.d.ts +1 -2
- package/dist/lib/agent-logs.js +1 -17
- package/dist/lib/agent-preflight.d.ts +1 -7
- package/dist/lib/agent-preflight.js +1 -77
- package/dist/lib/agent-render.d.ts +1 -8
- package/dist/lib/agent-render.js +1 -218
- package/dist/lib/agent-shells.d.ts +1 -69
- package/dist/lib/agent-shells.js +1 -1021
- package/dist/lib/agent-status.d.ts +1 -3
- package/dist/lib/agent-status.js +1 -58
- package/dist/lib/agent-types.d.ts +1 -30
- package/dist/lib/agent-types.js +1 -1
- package/dist/lib/agents.d.ts +1 -6
- package/dist/lib/agents.js +1 -5
- package/dist/lib/builtin-compiled-workflow.d.ts +1 -38
- package/dist/lib/builtin-compiled-workflow.js +1 -94
- package/dist/lib/chart-guidance.d.ts +1 -1
- package/dist/lib/chart-guidance.js +1 -8
- package/dist/lib/compiled-compile.d.ts +1 -48
- package/dist/lib/compiled-compile.js +1 -255
- package/dist/lib/compiled-paths.d.ts +1 -40
- package/dist/lib/compiled-paths.js +3 -106
- package/dist/lib/compiled-raw.d.ts +1 -49
- package/dist/lib/compiled-raw.js +3 -102
- package/dist/lib/compiled-reset.d.ts +1 -2
- package/dist/lib/compiled-reset.js +3 -72
- package/dist/lib/compiled-schema.d.ts +1 -31
- package/dist/lib/compiled-schema.js +1 -141
- package/dist/lib/discovery.d.ts +1 -7
- package/dist/lib/discovery.js +1 -84
- package/dist/lib/execution-profile.d.ts +1 -17
- package/dist/lib/execution-profile.js +1 -84
- package/dist/lib/executors.d.ts +1 -32
- package/dist/lib/executors.js +1 -43
- package/dist/lib/filesystem.d.ts +1 -2
- package/dist/lib/filesystem.js +1 -55
- package/dist/lib/interf-bootstrap.d.ts +1 -3
- package/dist/lib/interf-bootstrap.js +3 -18
- package/dist/lib/interf-detect.d.ts +1 -33
- package/dist/lib/interf-detect.js +3 -176
- package/dist/lib/interf-scaffold.d.ts +1 -2
- package/dist/lib/interf-scaffold.js +3 -114
- package/dist/lib/interf-workflow-package.d.ts +1 -25
- package/dist/lib/interf-workflow-package.js +1 -342
- package/dist/lib/interf.d.ts +1 -5
- package/dist/lib/interf.js +3 -4
- package/dist/lib/local-workflows.d.ts +1 -54
- package/dist/lib/local-workflows.js +1 -422
- package/dist/lib/logger.d.ts +1 -3
- package/dist/lib/logger.js +1 -10
- package/dist/lib/package-root.d.ts +1 -0
- package/dist/lib/package-root.js +1 -0
- package/dist/lib/parse.d.ts +1 -8
- package/dist/lib/parse.js +1 -145
- package/dist/lib/project-paths.d.ts +1 -11
- package/dist/lib/project-paths.js +3 -32
- package/dist/lib/runtime-acceptance.d.ts +1 -9
- package/dist/lib/runtime-acceptance.js +1 -269
- package/dist/lib/runtime-contracts.d.ts +1 -2
- package/dist/lib/runtime-contracts.js +1 -48
- package/dist/lib/runtime-inventory.d.ts +1 -7
- package/dist/lib/runtime-inventory.js +1 -29
- package/dist/lib/runtime-paths.d.ts +1 -8
- package/dist/lib/runtime-paths.js +1 -26
- package/dist/lib/runtime-prompt.d.ts +1 -2
- package/dist/lib/runtime-prompt.js +1 -48
- package/dist/lib/runtime-reconcile.d.ts +1 -2
- package/dist/lib/runtime-reconcile.js +1 -193
- package/dist/lib/runtime-runs.d.ts +1 -11
- package/dist/lib/runtime-runs.js +1 -262
- package/dist/lib/runtime-types.d.ts +1 -43
- package/dist/lib/runtime-types.js +1 -1
- package/dist/lib/runtime.d.ts +1 -6
- package/dist/lib/runtime.js +1 -5
- package/dist/lib/schema.d.ts +4 -1073
- package/dist/lib/schema.js +6 -542
- package/dist/lib/source-config.d.ts +1 -39
- package/dist/lib/source-config.js +3 -293
- package/dist/lib/state-artifacts.d.ts +1 -8
- package/dist/lib/state-artifacts.js +1 -13
- package/dist/lib/state-health.d.ts +1 -4
- package/dist/lib/state-health.js +1 -132
- package/dist/lib/state-io.d.ts +1 -10
- package/dist/lib/state-io.js +1 -76
- package/dist/lib/state-paths.d.ts +1 -4
- package/dist/lib/state-paths.js +1 -13
- package/dist/lib/state-view.d.ts +1 -4
- package/dist/lib/state-view.js +1 -103
- package/dist/lib/state.d.ts +1 -7
- package/dist/lib/state.js +1 -12
- package/dist/lib/test-execution.d.ts +1 -14
- package/dist/lib/test-execution.js +3 -525
- package/dist/lib/test-matrices.d.ts +1 -90
- package/dist/lib/test-matrices.js +3 -96
- package/dist/lib/test-paths.d.ts +1 -12
- package/dist/lib/test-paths.js +3 -59
- package/dist/lib/test-profile-presets.d.ts +1 -57
- package/dist/lib/test-profile-presets.js +3 -50
- package/dist/lib/test-sandbox.d.ts +1 -11
- package/dist/lib/test-sandbox.js +3 -105
- package/dist/lib/test-specs.d.ts +1 -7
- package/dist/lib/test-specs.js +3 -114
- package/dist/lib/test-targets.d.ts +1 -5
- package/dist/lib/test-targets.js +3 -38
- package/dist/lib/test-types.d.ts +1 -17
- package/dist/lib/test-types.js +3 -1
- package/dist/lib/test.d.ts +1 -4
- package/dist/lib/test.js +3 -3
- package/dist/lib/util.d.ts +1 -4
- package/dist/lib/util.js +1 -25
- package/dist/lib/validate-compiled.d.ts +1 -27
- package/dist/lib/validate-compiled.js +1 -241
- package/dist/lib/validate-helpers.d.ts +1 -12
- package/dist/lib/validate-helpers.js +1 -41
- package/dist/lib/validate.d.ts +1 -21
- package/dist/lib/validate.js +1 -249
- package/dist/lib/workflow-authoring.d.ts +1 -26
- package/dist/lib/workflow-authoring.js +1 -119
- package/dist/lib/workflow-definitions.d.ts +1 -78
- package/dist/lib/workflow-definitions.js +1 -203
- package/dist/lib/workflow-edit-session.d.ts +1 -16
- package/dist/lib/workflow-edit-session.js +1 -57
- package/dist/lib/workflow-edit-utils.d.ts +1 -10
- package/dist/lib/workflow-edit-utils.js +1 -39
- package/dist/lib/workflow-helpers.d.ts +1 -38
- package/dist/lib/workflow-helpers.js +1 -167
- package/dist/lib/workflow-improvement.d.ts +1 -22
- package/dist/lib/workflow-improvement.js +1 -209
- package/dist/lib/workflow-primitives.d.ts +1 -2
- package/dist/lib/workflow-primitives.js +1 -5
- package/dist/lib/workflow-review-paths.d.ts +1 -10
- package/dist/lib/workflow-review-paths.js +1 -27
- package/dist/lib/workflow-stage-policy.d.ts +1 -5
- package/dist/lib/workflow-stage-policy.js +1 -31
- package/dist/lib/workflow-stage-runner.d.ts +1 -41
- package/dist/lib/workflow-stage-runner.js +1 -109
- package/dist/lib/workflows.d.ts +1 -15
- package/dist/lib/workflows.js +1 -31
- package/dist/packages/agents/index.d.ts +18 -0
- package/dist/packages/agents/index.js +16 -0
- package/dist/packages/agents/lib/agents.d.ts +6 -0
- package/dist/packages/agents/lib/agents.js +5 -0
- package/dist/packages/agents/lib/args.d.ts +4 -0
- package/dist/packages/agents/lib/args.js +52 -0
- package/dist/packages/agents/lib/chart-guidance.d.ts +1 -0
- package/dist/packages/agents/lib/chart-guidance.js +8 -0
- package/dist/packages/agents/lib/compiled-bootstrap.d.ts +3 -0
- package/dist/packages/agents/lib/compiled-bootstrap.js +18 -0
- package/dist/packages/agents/lib/constants.d.ts +5 -0
- package/dist/packages/agents/lib/constants.js +28 -0
- package/dist/packages/agents/lib/detection.d.ts +7 -0
- package/dist/packages/agents/lib/detection.js +65 -0
- package/dist/packages/agents/lib/execution-profile.d.ts +17 -0
- package/dist/packages/agents/lib/execution-profile.js +84 -0
- package/dist/packages/agents/lib/execution.d.ts +2 -0
- package/dist/packages/agents/lib/execution.js +243 -0
- package/dist/packages/agents/lib/executors.d.ts +32 -0
- package/dist/packages/agents/lib/executors.js +45 -0
- package/dist/packages/agents/lib/logs.d.ts +2 -0
- package/dist/packages/agents/lib/logs.js +17 -0
- package/dist/packages/agents/lib/preflight.d.ts +7 -0
- package/dist/packages/agents/lib/preflight.js +77 -0
- package/dist/packages/agents/lib/render.d.ts +8 -0
- package/dist/packages/agents/lib/render.js +218 -0
- package/dist/packages/agents/lib/schema.d.ts +8 -0
- package/dist/packages/agents/lib/schema.js +7 -0
- package/dist/packages/agents/lib/shells.d.ts +71 -0
- package/dist/packages/agents/lib/shells.js +1023 -0
- package/dist/packages/agents/lib/status.d.ts +3 -0
- package/dist/packages/agents/lib/status.js +58 -0
- package/dist/packages/agents/lib/types.d.ts +30 -0
- package/dist/packages/agents/lib/types.js +1 -0
- package/dist/{lib → packages/agents/lib}/user-config.d.ts +1 -0
- package/dist/{lib → packages/agents/lib}/user-config.js +3 -2
- package/dist/packages/compiler/compiled-compile.d.ts +4 -0
- package/dist/packages/compiler/compiled-compile.js +4 -0
- package/dist/packages/compiler/compiled-paths.d.ts +40 -0
- package/dist/packages/compiler/compiled-paths.js +106 -0
- package/dist/packages/compiler/compiled-pipeline.d.ts +39 -0
- package/dist/packages/compiler/compiled-pipeline.js +134 -0
- package/dist/packages/compiler/compiled-schema.d.ts +31 -0
- package/dist/packages/compiler/compiled-schema.js +141 -0
- package/dist/packages/compiler/compiled-stage-plan.d.ts +15 -0
- package/dist/packages/compiler/compiled-stage-plan.js +79 -0
- package/dist/packages/compiler/compiled-stage-runner.d.ts +10 -0
- package/dist/packages/compiler/compiled-stage-runner.js +46 -0
- package/dist/packages/compiler/compiled-target.d.ts +11 -0
- package/dist/packages/compiler/compiled-target.js +16 -0
- package/dist/packages/compiler/discovery.d.ts +7 -0
- package/dist/packages/compiler/discovery.js +80 -0
- package/dist/packages/compiler/index.d.ts +24 -0
- package/dist/packages/compiler/index.js +23 -0
- package/dist/packages/compiler/lib/schema.d.ts +684 -0
- package/dist/packages/compiler/lib/schema.js +363 -0
- package/dist/packages/compiler/raw-snapshot.d.ts +49 -0
- package/dist/packages/compiler/raw-snapshot.js +102 -0
- package/dist/packages/compiler/reset.d.ts +2 -0
- package/dist/packages/compiler/reset.js +72 -0
- package/dist/packages/compiler/runtime-acceptance.d.ts +9 -0
- package/dist/packages/compiler/runtime-acceptance.js +269 -0
- package/dist/packages/compiler/runtime-contracts.d.ts +2 -0
- package/dist/packages/compiler/runtime-contracts.js +48 -0
- package/dist/packages/compiler/runtime-inventory.d.ts +7 -0
- package/dist/packages/compiler/runtime-inventory.js +29 -0
- package/dist/packages/compiler/runtime-paths.d.ts +8 -0
- package/dist/packages/compiler/runtime-paths.js +26 -0
- package/dist/packages/compiler/runtime-prompt.d.ts +2 -0
- package/dist/packages/compiler/runtime-prompt.js +48 -0
- package/dist/packages/compiler/runtime-reconcile.d.ts +2 -0
- package/dist/packages/compiler/runtime-reconcile.js +193 -0
- package/dist/packages/compiler/runtime-runs.d.ts +11 -0
- package/dist/packages/compiler/runtime-runs.js +262 -0
- package/dist/packages/compiler/runtime-types.d.ts +43 -0
- package/dist/packages/compiler/runtime-types.js +1 -0
- package/dist/packages/compiler/runtime.d.ts +6 -0
- package/dist/packages/compiler/runtime.js +5 -0
- package/dist/packages/compiler/state-artifacts.d.ts +8 -0
- package/dist/packages/compiler/state-artifacts.js +13 -0
- package/dist/packages/compiler/state-health.d.ts +4 -0
- package/dist/packages/compiler/state-health.js +132 -0
- package/dist/packages/compiler/state-io.d.ts +10 -0
- package/dist/packages/compiler/state-io.js +76 -0
- package/dist/packages/compiler/state-paths.d.ts +4 -0
- package/dist/packages/compiler/state-paths.js +13 -0
- package/dist/packages/compiler/state-view.d.ts +4 -0
- package/dist/packages/compiler/state-view.js +103 -0
- package/dist/packages/compiler/state.d.ts +7 -0
- package/dist/packages/compiler/state.js +12 -0
- package/dist/packages/compiler/validate-compiled.d.ts +27 -0
- package/dist/packages/compiler/validate-compiled.js +241 -0
- package/dist/packages/compiler/validate-helpers.d.ts +12 -0
- package/dist/packages/compiler/validate-helpers.js +41 -0
- package/dist/packages/compiler/validate.d.ts +21 -0
- package/dist/packages/compiler/validate.js +249 -0
- package/dist/packages/compiler/workflow-primitives.d.ts +2 -0
- package/dist/packages/compiler/workflow-primitives.js +5 -0
- package/dist/packages/compiler/workflows.d.ts +15 -0
- package/dist/packages/compiler/workflows.js +31 -0
- package/dist/packages/project-model/compiled-paths.d.ts +1 -0
- package/dist/packages/project-model/compiled-paths.js +1 -0
- package/dist/packages/project-model/compiled-raw.d.ts +1 -0
- package/dist/packages/project-model/compiled-raw.js +1 -0
- package/dist/packages/project-model/compiled-reset.d.ts +1 -0
- package/dist/packages/project-model/compiled-reset.js +1 -0
- package/dist/packages/project-model/index.d.ts +11 -0
- package/dist/packages/project-model/index.js +10 -0
- package/dist/packages/project-model/interf-bootstrap.d.ts +1 -0
- package/dist/packages/project-model/interf-bootstrap.js +1 -0
- package/dist/packages/project-model/interf-detect.d.ts +33 -0
- package/dist/packages/project-model/interf-detect.js +176 -0
- package/dist/packages/project-model/interf-scaffold.d.ts +2 -0
- package/dist/packages/project-model/interf-scaffold.js +124 -0
- package/dist/packages/project-model/interf.d.ts +5 -0
- package/dist/packages/project-model/interf.js +4 -0
- package/dist/packages/project-model/lib/schema.d.ts +125 -0
- package/dist/packages/project-model/lib/schema.js +62 -0
- package/dist/packages/project-model/project-paths.d.ts +11 -0
- package/dist/packages/project-model/project-paths.js +32 -0
- package/dist/packages/project-model/source-config.d.ts +38 -0
- package/dist/packages/project-model/source-config.js +298 -0
- package/dist/packages/shared/file-types.d.ts +1 -0
- package/dist/packages/shared/file-types.js +4 -0
- package/dist/packages/shared/filesystem.d.ts +2 -0
- package/dist/packages/shared/filesystem.js +55 -0
- package/dist/packages/shared/index.d.ts +7 -0
- package/dist/packages/shared/index.js +7 -0
- package/dist/packages/shared/logger.d.ts +3 -0
- package/dist/packages/shared/logger.js +10 -0
- package/dist/packages/shared/naming.d.ts +1 -0
- package/dist/packages/shared/naming.js +8 -0
- package/dist/packages/shared/parse.d.ts +8 -0
- package/dist/packages/shared/parse.js +145 -0
- package/dist/packages/shared/path-guards.d.ts +2 -0
- package/dist/packages/shared/path-guards.js +14 -0
- package/dist/packages/shared/util.d.ts +3 -0
- package/dist/packages/shared/util.js +3 -0
- package/dist/packages/testing/index.d.ts +13 -0
- package/dist/packages/testing/index.js +10 -0
- package/dist/packages/testing/lib/schema.d.ts +261 -0
- package/dist/packages/testing/lib/schema.js +119 -0
- package/dist/packages/testing/test-execution.d.ts +14 -0
- package/dist/packages/testing/test-execution.js +525 -0
- package/dist/packages/testing/test-matrices.d.ts +90 -0
- package/dist/packages/testing/test-matrices.js +96 -0
- package/dist/packages/testing/test-paths.d.ts +12 -0
- package/dist/packages/testing/test-paths.js +59 -0
- package/dist/packages/testing/test-profile-presets.d.ts +57 -0
- package/dist/packages/testing/test-profile-presets.js +50 -0
- package/dist/packages/testing/test-sandbox.d.ts +11 -0
- package/dist/packages/testing/test-sandbox.js +105 -0
- package/dist/packages/testing/test-specs.d.ts +7 -0
- package/dist/packages/testing/test-specs.js +114 -0
- package/dist/packages/testing/test-targets.d.ts +5 -0
- package/dist/packages/testing/test-targets.js +38 -0
- package/dist/packages/testing/test-types.d.ts +16 -0
- package/dist/packages/testing/test-types.js +1 -0
- package/dist/packages/testing/test.d.ts +4 -0
- package/dist/packages/testing/test.js +3 -0
- package/dist/packages/workflow-authoring/index.d.ts +4 -0
- package/dist/packages/workflow-authoring/index.js +4 -0
- package/dist/packages/workflow-authoring/lib/workflow-edit-utils.d.ts +10 -0
- package/dist/packages/workflow-authoring/lib/workflow-edit-utils.js +39 -0
- package/dist/packages/workflow-authoring/workflow-authoring.d.ts +26 -0
- package/dist/packages/workflow-authoring/workflow-authoring.js +121 -0
- package/dist/packages/workflow-authoring/workflow-edit-session.d.ts +16 -0
- package/dist/packages/workflow-authoring/workflow-edit-session.js +57 -0
- package/dist/packages/workflow-authoring/workflow-improvement.d.ts +23 -0
- package/dist/packages/workflow-authoring/workflow-improvement.js +210 -0
- package/dist/packages/workflow-package/builtin-compiled-workflow.d.ts +38 -0
- package/dist/packages/workflow-package/builtin-compiled-workflow.js +94 -0
- package/dist/packages/workflow-package/context-interface.d.ts +96 -0
- package/dist/packages/workflow-package/context-interface.js +146 -0
- package/dist/packages/workflow-package/index.d.ts +11 -0
- package/dist/packages/workflow-package/index.js +11 -0
- package/dist/packages/workflow-package/interf-workflow-package.d.ts +25 -0
- package/dist/packages/workflow-package/interf-workflow-package.js +389 -0
- package/dist/{lib/config.d.ts → packages/workflow-package/lib/package-root.d.ts} +0 -1
- package/dist/packages/workflow-package/lib/package-root.js +6 -0
- package/dist/packages/workflow-package/local-workflows.d.ts +57 -0
- package/dist/packages/workflow-package/local-workflows.js +424 -0
- package/dist/packages/workflow-package/workflow-definitions.d.ts +82 -0
- package/dist/packages/workflow-package/workflow-definitions.js +210 -0
- package/dist/packages/workflow-package/workflow-helpers.d.ts +38 -0
- package/dist/packages/workflow-package/workflow-helpers.js +167 -0
- package/dist/packages/workflow-package/workflow-review-paths.d.ts +10 -0
- package/dist/packages/workflow-package/workflow-review-paths.js +27 -0
- package/dist/packages/workflow-package/workflow-stage-policy.d.ts +5 -0
- package/dist/packages/workflow-package/workflow-stage-policy.js +31 -0
- package/dist/packages/workflow-package/workflow-stage-runner.d.ts +41 -0
- package/dist/packages/workflow-package/workflow-stage-runner.js +109 -0
- package/package.json +43 -19
- package/dist/commands/test.d.ts +0 -3
- package/dist/lib/compiled-home.d.ts +0 -5
- package/dist/lib/compiled-home.js +0 -32
- package/dist/lib/config.js +0 -9
- /package/dist/{commands → cli/commands}/dataset-selection.js +0 -0
- /package/dist/{commands → cli/commands}/default.d.ts +0 -0
- /package/dist/{commands → cli/commands}/doctor.d.ts +0 -0
- /package/dist/{commands → cli/commands}/list.d.ts +0 -0
- /package/dist/{commands → cli/commands}/reset.d.ts +0 -0
- /package/dist/{commands → cli/commands}/status.d.ts +0 -0
- /package/dist/{commands → cli/commands}/verify.d.ts +0 -0
package/dist/lib/state-view.js
CHANGED
|
@@ -1,103 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
-
import { discoverSourceFiles } from "./discovery.js";
|
|
4
|
-
import { readInterfConfig, resolveSourceFolderPath } from "./interf.js";
|
|
5
|
-
import { loadCompiledViewSpec, saveCompiledViewSpec, } from "./state-io.js";
|
|
6
|
-
import { rawSnapshotPath } from "./state-paths.js";
|
|
7
|
-
import { compiledRuntimeRoot, workflowPackagePathForCompiled } from "./compiled-paths.js";
|
|
8
|
-
import { readCompiledSchemaFile } from "./compiled-schema.js";
|
|
9
|
-
import { getActiveCompiledWorkflow, resolveRequiredCompiledWorkflowFromConfig } from "./workflow-definitions.js";
|
|
10
|
-
export function ensureCompiledViewSpec(dirPath) {
|
|
11
|
-
const existing = loadCompiledViewSpec(dirPath);
|
|
12
|
-
const now = new Date().toISOString();
|
|
13
|
-
const compiledName = readInterfConfig(dirPath)?.name ?? basename(dirPath);
|
|
14
|
-
const defaults = buildDefaultCompiledViewSpec(dirPath, compiledName, now);
|
|
15
|
-
if (existing) {
|
|
16
|
-
const normalized = normalizeCompiledViewSpec(existing, defaults);
|
|
17
|
-
if (JSON.stringify(normalized) !== JSON.stringify(existing)) {
|
|
18
|
-
saveCompiledViewSpec(dirPath, normalized);
|
|
19
|
-
}
|
|
20
|
-
return normalized;
|
|
21
|
-
}
|
|
22
|
-
saveCompiledViewSpec(dirPath, defaults);
|
|
23
|
-
return defaults;
|
|
24
|
-
}
|
|
25
|
-
export function ensureCompiledRawSnapshot(dirPath) {
|
|
26
|
-
const now = new Date().toISOString();
|
|
27
|
-
const config = readInterfConfig(dirPath);
|
|
28
|
-
const compiledName = config?.name ?? basename(dirPath);
|
|
29
|
-
const sourcePath = resolveSourceFolderPath(dirPath, config);
|
|
30
|
-
const discovery = discoverSourceFiles(sourcePath, dirPath);
|
|
31
|
-
const sampleFiles = [...discovery.sourceFiles]
|
|
32
|
-
.sort((a, b) => a.length - b.length || a.localeCompare(b))
|
|
33
|
-
.slice(0, 10);
|
|
34
|
-
const sourceRoot = relative(dirPath, sourcePath) || ".";
|
|
35
|
-
const artifact = {
|
|
36
|
-
kind: "compiled-raw-snapshot",
|
|
37
|
-
version: 1,
|
|
38
|
-
generated_at: now,
|
|
39
|
-
target_name: compiledName,
|
|
40
|
-
source_root: sourceRoot,
|
|
41
|
-
raw_root: sourceRoot,
|
|
42
|
-
source_total: discovery.totalCount,
|
|
43
|
-
sample_files: sampleFiles,
|
|
44
|
-
};
|
|
45
|
-
mkdirSync(compiledRuntimeRoot(dirPath), { recursive: true });
|
|
46
|
-
writeFileSync(rawSnapshotPath(dirPath), JSON.stringify(artifact, null, 2) + "\n");
|
|
47
|
-
return artifact;
|
|
48
|
-
}
|
|
49
|
-
export function normalizeCompiledViewSpec(existing, defaults) {
|
|
50
|
-
const resolvedDefaults = defaults ?? existing;
|
|
51
|
-
return {
|
|
52
|
-
...resolvedDefaults,
|
|
53
|
-
...existing,
|
|
54
|
-
cards: Array.isArray(existing.cards) && existing.cards.length > 0
|
|
55
|
-
? existing.cards.map((card) => {
|
|
56
|
-
if (card.format !== "fraction" || card.denominator_metric)
|
|
57
|
-
return card;
|
|
58
|
-
const defaultCard = resolvedDefaults.cards.find((candidate) => candidate.id === card.id);
|
|
59
|
-
return {
|
|
60
|
-
...card,
|
|
61
|
-
denominator_metric: defaultCard?.denominator_metric ?? "stage_total",
|
|
62
|
-
};
|
|
63
|
-
})
|
|
64
|
-
: resolvedDefaults.cards,
|
|
65
|
-
sections: Array.isArray(existing.sections) && existing.sections.length > 0 ? existing.sections : resolvedDefaults.sections,
|
|
66
|
-
graph_scope: Array.isArray(existing.graph_scope) && existing.graph_scope.length > 0
|
|
67
|
-
? existing.graph_scope
|
|
68
|
-
: resolvedDefaults.graph_scope,
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
function buildDefaultCompiledViewSpec(dirPath, compiledName, generatedAt) {
|
|
72
|
-
const config = readInterfConfig(dirPath);
|
|
73
|
-
const workflowId = resolveRequiredCompiledWorkflowFromConfig(config, `.interf/interf.json for ${dirPath}`);
|
|
74
|
-
const workflow = getActiveCompiledWorkflow(dirPath, workflowId);
|
|
75
|
-
const schema = readCompiledSchemaFile(workflowPackagePathForCompiled(dirPath));
|
|
76
|
-
const outputDocumentPaths = (schema?.zones ?? [])
|
|
77
|
-
.filter((zone) => zone.role === "output" && zone.kind !== "runtime")
|
|
78
|
-
.map((zone) => zone.path);
|
|
79
|
-
const defaultNote = (schema?.zones ?? []).find((zone) => zone.role === "output" && zone.kind === "file")?.path ?? null;
|
|
80
|
-
return {
|
|
81
|
-
kind: "compiled-view-spec",
|
|
82
|
-
version: 2,
|
|
83
|
-
generated_at: generatedAt,
|
|
84
|
-
target_name: compiledName,
|
|
85
|
-
health_source: ".interf/runtime/health.json",
|
|
86
|
-
state_source: ".interf/runtime/state.json",
|
|
87
|
-
default_note: defaultNote,
|
|
88
|
-
graph_scope: outputDocumentPaths,
|
|
89
|
-
cards: [
|
|
90
|
-
{ id: "source-total", label: "Source Files", metric: "source_total", format: "number" },
|
|
91
|
-
{ id: "stage-total", label: "Stages", metric: "stage_total", format: "number" },
|
|
92
|
-
{ id: "completed-stages", label: "Completed", metric: "completed_stages", denominator_metric: "stage_total", format: "fraction" },
|
|
93
|
-
],
|
|
94
|
-
sections: [
|
|
95
|
-
{ id: "status", type: "status", title: "Compile Status", path: ".interf/runtime/health.json" },
|
|
96
|
-
{ id: "cards", type: "cards", title: "Key Metrics", path: ".interf/runtime/health.json" },
|
|
97
|
-
...(outputDocumentPaths.length > 0
|
|
98
|
-
? [{ id: "outputs", type: "documents", title: "Output Zones", paths: outputDocumentPaths }]
|
|
99
|
-
: []),
|
|
100
|
-
{ id: "workflow", type: "documents", title: "Workflow Docs", paths: ["workflow/README.md", ...workflow.stages.map((stage) => `workflow/compile/stages/${stage.skillDir}`)] },
|
|
101
|
-
],
|
|
102
|
-
};
|
|
103
|
-
}
|
|
1
|
+
export * from "../packages/compiler/state-view.js";
|
package/dist/lib/state.d.ts
CHANGED
|
@@ -1,7 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
export { statePath, healthPath, viewSpecPath, rawSnapshotPath, } from "./state-paths.js";
|
|
3
|
-
export { loadState, saveState, loadCompiledHealth, saveCompiledHealth, loadCompiledViewSpec, loadCompiledRawSnapshot, saveCompiledViewSpec, initCompiledState, } from "./state-io.js";
|
|
4
|
-
export { refreshCompiledArtifacts, } from "./state-artifacts.js";
|
|
5
|
-
export { computeCompiledHealth, resolveCompiledStatus, resolveCompiledStage, } from "./state-health.js";
|
|
6
|
-
export { ensureCompiledViewSpec, ensureCompiledRawSnapshot, normalizeCompiledViewSpec, } from "./state-view.js";
|
|
7
|
-
export declare function initializeCompiledRuntimeState(dirPath: string): void;
|
|
1
|
+
export * from "../packages/compiler/state.js";
|
package/dist/lib/state.js
CHANGED
|
@@ -1,12 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
export { loadState, saveState, loadCompiledHealth, saveCompiledHealth, loadCompiledViewSpec, loadCompiledRawSnapshot, saveCompiledViewSpec, initCompiledState, } from "./state-io.js";
|
|
3
|
-
export { refreshCompiledArtifacts, } from "./state-artifacts.js";
|
|
4
|
-
export { computeCompiledHealth, resolveCompiledStatus, resolveCompiledStage, } from "./state-health.js";
|
|
5
|
-
export { ensureCompiledViewSpec, ensureCompiledRawSnapshot, normalizeCompiledViewSpec, } from "./state-view.js";
|
|
6
|
-
import { saveEmptyCompiledInventory, initCompiledState, saveState } from "./state-io.js";
|
|
7
|
-
import { refreshCompiledArtifacts } from "./state-artifacts.js";
|
|
8
|
-
export function initializeCompiledRuntimeState(dirPath) {
|
|
9
|
-
saveState(dirPath, initCompiledState());
|
|
10
|
-
saveEmptyCompiledInventory(dirPath);
|
|
11
|
-
refreshCompiledArtifacts(dirPath, { ensureViewSpec: true });
|
|
12
|
-
}
|
|
1
|
+
export * from "../packages/compiler/state.js";
|
|
@@ -1,14 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import type { TestTargetRun, TestTargetCandidate, LoadedTestSpec } from "./test-types.js";
|
|
3
|
-
import { type TestSandboxRetentionMode } from "./test-sandbox.js";
|
|
4
|
-
export declare function runTargetTests(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[]): TestTargetRun;
|
|
5
|
-
export declare function runTargetTestsWithJudge(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], executor: WorkflowExecutor, options?: {
|
|
6
|
-
preserveSandboxes?: TestSandboxRetentionMode;
|
|
7
|
-
artifactRootPath?: string;
|
|
8
|
-
}): Promise<TestTargetRun>;
|
|
9
|
-
export declare function runTargetTestsAuto(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], options?: {
|
|
10
|
-
executor?: WorkflowExecutor | null;
|
|
11
|
-
preserveSandboxes?: TestSandboxRetentionMode;
|
|
12
|
-
artifactRootPath?: string;
|
|
13
|
-
}): Promise<TestTargetRun>;
|
|
14
|
-
export declare function saveTargetTestRun(artifactRootPath: string, result: TestTargetRun): string;
|
|
1
|
+
export * from "../packages/testing/test-execution.js";
|
|
@@ -1,525 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import { buildRuntimeExecutorInfo } from "./executors.js";
|
|
5
|
-
import { targetTestRunGitignorePath, targetTestRunsPath, targetTestSandboxGitignorePath, targetTestSandboxesPath, normalizeTestId, } from "./test-paths.js";
|
|
6
|
-
import { createTestSandbox, } from "./test-sandbox.js";
|
|
7
|
-
function parseWords(content) {
|
|
8
|
-
return content.trim().split(/\s+/).filter(Boolean).length;
|
|
9
|
-
}
|
|
10
|
-
function normalizeText(content) {
|
|
11
|
-
return content.toLowerCase().replace(/\s+/g, " ").trim();
|
|
12
|
-
}
|
|
13
|
-
function evaluateTextExpect(testCase, content) {
|
|
14
|
-
const checks = [];
|
|
15
|
-
const normalized = normalizeText(content);
|
|
16
|
-
const wordCount = parseWords(content);
|
|
17
|
-
for (const phrase of testCase.expect?.must_include ?? []) {
|
|
18
|
-
const ok = normalized.includes(normalizeText(phrase));
|
|
19
|
-
checks.push({
|
|
20
|
-
label: `must include "${phrase}"`,
|
|
21
|
-
ok,
|
|
22
|
-
detail: ok ? "present" : "missing",
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
for (const options of testCase.expect?.must_include_one_of ?? []) {
|
|
26
|
-
const ok = options.some((phrase) => normalized.includes(normalizeText(phrase)));
|
|
27
|
-
checks.push({
|
|
28
|
-
label: `must include one of [${options.map((phrase) => `"${phrase}"`).join(", ")}]`,
|
|
29
|
-
ok,
|
|
30
|
-
detail: ok ? "present" : "missing",
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
for (const phrase of testCase.expect?.must_not_include ?? []) {
|
|
34
|
-
const ok = !normalized.includes(normalizeText(phrase));
|
|
35
|
-
checks.push({
|
|
36
|
-
label: `must not include "${phrase}"`,
|
|
37
|
-
ok,
|
|
38
|
-
detail: ok ? "absent" : "present",
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
if (typeof testCase.expect?.min_words === "number") {
|
|
42
|
-
const ok = wordCount >= testCase.expect.min_words;
|
|
43
|
-
checks.push({
|
|
44
|
-
label: `min words ${testCase.expect.min_words}`,
|
|
45
|
-
ok,
|
|
46
|
-
detail: `${wordCount} words`,
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
if (typeof testCase.expect?.max_words === "number") {
|
|
50
|
-
const ok = wordCount <= testCase.expect.max_words;
|
|
51
|
-
checks.push({
|
|
52
|
-
label: `max words ${testCase.expect.max_words}`,
|
|
53
|
-
ok,
|
|
54
|
-
detail: `${wordCount} words`,
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
checks,
|
|
59
|
-
wordCount,
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
function testCaseNeedsExecutor(testCase) {
|
|
63
|
-
return !testCase.file || Boolean(testCase.answer);
|
|
64
|
-
}
|
|
65
|
-
function runTestCase(target, testCase) {
|
|
66
|
-
if (!testCase.file) {
|
|
67
|
-
throw new Error(`Test case "${testCase.id}" requires an executor because it has no file target.`);
|
|
68
|
-
}
|
|
69
|
-
const outputPath = join(target.path, testCase.file);
|
|
70
|
-
const checks = [];
|
|
71
|
-
if (!existsSync(outputPath)) {
|
|
72
|
-
checks.push({
|
|
73
|
-
label: "file exists",
|
|
74
|
-
ok: false,
|
|
75
|
-
detail: `Missing ${testCase.file}`,
|
|
76
|
-
});
|
|
77
|
-
return {
|
|
78
|
-
caseId: testCase.id,
|
|
79
|
-
question: testCase.question,
|
|
80
|
-
...(testCase.file ? { file: testCase.file } : {}),
|
|
81
|
-
ok: false,
|
|
82
|
-
wordCount: 0,
|
|
83
|
-
passedChecks: 0,
|
|
84
|
-
totalChecks: checks.length,
|
|
85
|
-
checks,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
checks.push({
|
|
89
|
-
label: "file exists",
|
|
90
|
-
ok: true,
|
|
91
|
-
detail: `Found ${testCase.file}`,
|
|
92
|
-
});
|
|
93
|
-
const content = readFileSync(outputPath, "utf8");
|
|
94
|
-
const evaluated = evaluateTextExpect(testCase, content);
|
|
95
|
-
const wordCount = evaluated.wordCount;
|
|
96
|
-
checks.push(...evaluated.checks);
|
|
97
|
-
const passedChecks = checks.filter((check) => check.ok).length;
|
|
98
|
-
return {
|
|
99
|
-
caseId: testCase.id,
|
|
100
|
-
question: testCase.question,
|
|
101
|
-
...(testCase.file ? { file: testCase.file } : {}),
|
|
102
|
-
ok: passedChecks === checks.length,
|
|
103
|
-
wordCount,
|
|
104
|
-
passedChecks,
|
|
105
|
-
totalChecks: checks.length,
|
|
106
|
-
checks,
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
function buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath) {
|
|
110
|
-
return [
|
|
111
|
-
"You are judging whether one Interf test answer passes.",
|
|
112
|
-
"Do not browse other files or ask follow-up questions.",
|
|
113
|
-
"Judge only from the truth-check rule and the candidate answer below.",
|
|
114
|
-
"Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
|
|
115
|
-
`Write JSON to ${JSON.stringify(verdictPath)} with keys: pass (boolean), summary (string).`,
|
|
116
|
-
"Before finishing, write the JSON verdict file.",
|
|
117
|
-
"Final line must be `DONE: pass=true - <short summary>` or `DONE: pass=false - <short summary>`.",
|
|
118
|
-
`Question: ${testCase.question}`,
|
|
119
|
-
`Expected answer: ${testCase.answer ?? "The answer clearly satisfies the question."}`,
|
|
120
|
-
`Strictness: ${testCase.strictness ?? "approximate"}`,
|
|
121
|
-
`Candidate: ${candidateLabel}`,
|
|
122
|
-
"Candidate answer starts after the next line and ends at `END CANDIDATE`.",
|
|
123
|
-
candidateContent,
|
|
124
|
-
"END CANDIDATE",
|
|
125
|
-
].join("\n");
|
|
126
|
-
}
|
|
127
|
-
function readTestJudgeVerdict(verdictPath) {
|
|
128
|
-
if (!existsSync(verdictPath))
|
|
129
|
-
return null;
|
|
130
|
-
const raw = JSON.parse(readFileSync(verdictPath, "utf8"));
|
|
131
|
-
return {
|
|
132
|
-
pass: raw.pass === true,
|
|
133
|
-
summary: typeof raw.summary === "string" ? raw.summary : "",
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
function readTestJudgeVerdictFromStatus(statusPath) {
|
|
137
|
-
if (!existsSync(statusPath))
|
|
138
|
-
return null;
|
|
139
|
-
const lines = readFileSync(statusPath, "utf8")
|
|
140
|
-
.split(/\r?\n/)
|
|
141
|
-
.map((line) => line.trim())
|
|
142
|
-
.filter((line) => line.length > 0);
|
|
143
|
-
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
144
|
-
const line = lines[index];
|
|
145
|
-
if (!line)
|
|
146
|
-
continue;
|
|
147
|
-
if (!/^(DONE|BLOCKED|ERROR):/i.test(line))
|
|
148
|
-
continue;
|
|
149
|
-
const normalized = line.toLowerCase();
|
|
150
|
-
const summary = line.replace(/^(DONE|BLOCKED|ERROR):\s*/i, "").trim();
|
|
151
|
-
if (normalized.includes("pass=true")) {
|
|
152
|
-
return { pass: true, summary };
|
|
153
|
-
}
|
|
154
|
-
if (normalized.includes("pass=false")) {
|
|
155
|
-
return { pass: false, summary };
|
|
156
|
-
}
|
|
157
|
-
if (/values match expected|matches expected|candidate matches|answer matches|expected values match/i.test(line)) {
|
|
158
|
-
return { pass: true, summary };
|
|
159
|
-
}
|
|
160
|
-
if (/does not match|do not match|mismatch|candidate fails|answer fails|expected values do not match/i.test(line)) {
|
|
161
|
-
return { pass: false, summary };
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
return null;
|
|
165
|
-
}
|
|
166
|
-
async function runTargetTestsJudge(testCase, executor, candidateLabel, candidateContent) {
|
|
167
|
-
const tempDir = mkdtempSync(join(tmpdir(), "interf-test-judge-"));
|
|
168
|
-
let executionError = null;
|
|
169
|
-
let verdict = null;
|
|
170
|
-
try {
|
|
171
|
-
const verdictPath = join(tempDir, "verdict.json");
|
|
172
|
-
const statusPath = join(tempDir, "judge.status.log");
|
|
173
|
-
const prompt = buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath);
|
|
174
|
-
try {
|
|
175
|
-
await executor.execute(tempDir, prompt, {
|
|
176
|
-
statusLogPath: statusPath,
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
catch (error) {
|
|
180
|
-
executionError = error instanceof Error ? error.message : String(error);
|
|
181
|
-
}
|
|
182
|
-
try {
|
|
183
|
-
verdict = readTestJudgeVerdict(verdictPath);
|
|
184
|
-
if (!verdict) {
|
|
185
|
-
verdict = readTestJudgeVerdictFromStatus(statusPath);
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
catch (error) {
|
|
189
|
-
executionError = error instanceof Error ? error.message : String(error);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
finally {
|
|
193
|
-
rmSync(tempDir, { recursive: true, force: true });
|
|
194
|
-
}
|
|
195
|
-
return { verdict, error: executionError };
|
|
196
|
-
}
|
|
197
|
-
function buildTestQueryPrompt(target, testCase, answerPath, tracePath) {
|
|
198
|
-
const header = target.type === "compiled"
|
|
199
|
-
? [
|
|
200
|
-
"You are running an Interf test inside an isolated sandboxed compiled dataset.",
|
|
201
|
-
"Read `AGENTS.md` first.",
|
|
202
|
-
"Use the local native `interf-query` skill available in this compiled dataset.",
|
|
203
|
-
"Answer the truth-check question the same way you would answer a real user inside this compiled dataset.",
|
|
204
|
-
"Prefer the workflow-declared compiled outputs before raw fallback.",
|
|
205
|
-
"This sandbox is self-contained: the copied compiled dataset has its own sanitized `raw/` fallback via `.interf/interf.json` `source.path`.",
|
|
206
|
-
"The original project root is intentionally absent from this sandbox. Work only from this sandboxed compiled dataset and its embedded raw files.",
|
|
207
|
-
]
|
|
208
|
-
: [
|
|
209
|
-
"You are running an Interf baseline test inside an isolated raw test shell.",
|
|
210
|
-
"Read `AGENTS.md` first.",
|
|
211
|
-
"Use the local native `interf-query` skill available in this shell.",
|
|
212
|
-
"There is no compiled dataset in this sandbox.",
|
|
213
|
-
"Answer only from `raw/` inside this shell.",
|
|
214
|
-
"The original project root is intentionally absent from this sandbox.",
|
|
215
|
-
];
|
|
216
|
-
return [
|
|
217
|
-
...header,
|
|
218
|
-
"Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
|
|
219
|
-
"Do not ask follow-up questions.",
|
|
220
|
-
`Write the answer to ${JSON.stringify(answerPath)}.`,
|
|
221
|
-
`Write the trace to ${JSON.stringify(tracePath)} with keys: case_id, target, artifacts_consulted, raw_paths_read, used_raw_fallback, answer_summary.`,
|
|
222
|
-
`Set \`case_id\` to ${JSON.stringify(testCase.id)}.`,
|
|
223
|
-
`Set \`target\` to ${JSON.stringify(target.type)}.`,
|
|
224
|
-
`Question: ${testCase.question}`,
|
|
225
|
-
].join("\n");
|
|
226
|
-
}
|
|
227
|
-
async function runLiveTestCase(target, testCase, executor) {
|
|
228
|
-
const tempDir = mkdtempSync(join(tmpdir(), "interf-test-live-"));
|
|
229
|
-
const answerPath = join(tempDir, "answer.md");
|
|
230
|
-
const tracePath = join(tempDir, "trace.json");
|
|
231
|
-
const statusPath = join(tempDir, "status.log");
|
|
232
|
-
const eventPath = join(tempDir, "events.ndjson");
|
|
233
|
-
const prompt = buildTestQueryPrompt(target, testCase, answerPath, tracePath);
|
|
234
|
-
let executionError = null;
|
|
235
|
-
let code = -1;
|
|
236
|
-
try {
|
|
237
|
-
try {
|
|
238
|
-
code = await executor.execute(target.path, prompt, {
|
|
239
|
-
eventLogPath: eventPath,
|
|
240
|
-
statusLogPath: statusPath,
|
|
241
|
-
});
|
|
242
|
-
}
|
|
243
|
-
catch (error) {
|
|
244
|
-
executionError = error instanceof Error ? error.message : String(error);
|
|
245
|
-
}
|
|
246
|
-
const checks = [];
|
|
247
|
-
if (!existsSync(answerPath)) {
|
|
248
|
-
checks.push({
|
|
249
|
-
label: "answer exists",
|
|
250
|
-
ok: false,
|
|
251
|
-
detail: executionError ? `missing answer file (${executionError})` : "missing answer file",
|
|
252
|
-
});
|
|
253
|
-
return {
|
|
254
|
-
caseId: testCase.id,
|
|
255
|
-
question: testCase.question,
|
|
256
|
-
ok: false,
|
|
257
|
-
wordCount: 0,
|
|
258
|
-
passedChecks: 0,
|
|
259
|
-
totalChecks: checks.length,
|
|
260
|
-
checks,
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
const answer = readFileSync(answerPath, "utf8");
|
|
264
|
-
const evaluated = evaluateTextExpect(testCase, answer);
|
|
265
|
-
checks.push({
|
|
266
|
-
label: "answer exists",
|
|
267
|
-
ok: true,
|
|
268
|
-
detail: "present",
|
|
269
|
-
});
|
|
270
|
-
checks.push(...evaluated.checks);
|
|
271
|
-
let trace = null;
|
|
272
|
-
if (existsSync(tracePath)) {
|
|
273
|
-
try {
|
|
274
|
-
trace = JSON.parse(readFileSync(tracePath, "utf8"));
|
|
275
|
-
checks.push({
|
|
276
|
-
label: "trace exists",
|
|
277
|
-
ok: true,
|
|
278
|
-
detail: "present",
|
|
279
|
-
});
|
|
280
|
-
}
|
|
281
|
-
catch (error) {
|
|
282
|
-
checks.push({
|
|
283
|
-
label: "trace exists",
|
|
284
|
-
ok: false,
|
|
285
|
-
detail: `invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
else {
|
|
290
|
-
checks.push({
|
|
291
|
-
label: "trace exists",
|
|
292
|
-
ok: false,
|
|
293
|
-
detail: "missing trace file",
|
|
294
|
-
});
|
|
295
|
-
}
|
|
296
|
-
if (testCase.answer) {
|
|
297
|
-
const judged = await runTargetTestsJudge(testCase, executor, `generated answer for ${testCase.id}`, answer);
|
|
298
|
-
checks.push({
|
|
299
|
-
label: "judge verdict",
|
|
300
|
-
ok: judged.verdict?.pass === true,
|
|
301
|
-
detail: judged.verdict?.summary && judged.verdict.summary.trim().length > 0
|
|
302
|
-
? judged.verdict.summary.trim()
|
|
303
|
-
: judged.error ?? "missing or invalid judge verdict",
|
|
304
|
-
});
|
|
305
|
-
}
|
|
306
|
-
const passedChecks = checks.filter((check) => check.ok).length;
|
|
307
|
-
return {
|
|
308
|
-
caseId: testCase.id,
|
|
309
|
-
question: testCase.question,
|
|
310
|
-
ok: code === 0 && passedChecks === checks.length,
|
|
311
|
-
wordCount: evaluated.wordCount,
|
|
312
|
-
passedChecks,
|
|
313
|
-
totalChecks: checks.length,
|
|
314
|
-
checks,
|
|
315
|
-
answer,
|
|
316
|
-
...(trace ? { trace } : {}),
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
finally {
|
|
320
|
-
rmSync(tempDir, { recursive: true, force: true });
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
async function runTestCaseWithJudge(target, testCase, executor) {
|
|
324
|
-
if (!testCase.file) {
|
|
325
|
-
return runLiveTestCase(target, testCase, executor);
|
|
326
|
-
}
|
|
327
|
-
const outputPath = join(target.path, testCase.file);
|
|
328
|
-
const checks = [];
|
|
329
|
-
if (!existsSync(outputPath)) {
|
|
330
|
-
checks.push({
|
|
331
|
-
label: "file exists",
|
|
332
|
-
ok: false,
|
|
333
|
-
detail: `Missing ${testCase.file}`,
|
|
334
|
-
});
|
|
335
|
-
return {
|
|
336
|
-
caseId: testCase.id,
|
|
337
|
-
question: testCase.question,
|
|
338
|
-
...(testCase.file ? { file: testCase.file } : {}),
|
|
339
|
-
ok: false,
|
|
340
|
-
wordCount: 0,
|
|
341
|
-
passedChecks: 0,
|
|
342
|
-
totalChecks: checks.length,
|
|
343
|
-
checks,
|
|
344
|
-
};
|
|
345
|
-
}
|
|
346
|
-
const content = readFileSync(outputPath, "utf8");
|
|
347
|
-
const evaluated = evaluateTextExpect(testCase, content);
|
|
348
|
-
const wordCount = evaluated.wordCount;
|
|
349
|
-
checks.push({
|
|
350
|
-
label: "file exists",
|
|
351
|
-
ok: true,
|
|
352
|
-
detail: `Found ${testCase.file}`,
|
|
353
|
-
});
|
|
354
|
-
checks.push(...evaluated.checks);
|
|
355
|
-
if (testCase.answer) {
|
|
356
|
-
const judged = await runTargetTestsJudge(testCase, executor, `compiled file ${outputPath}`, content);
|
|
357
|
-
checks.push({
|
|
358
|
-
label: "judge verdict",
|
|
359
|
-
ok: judged.verdict?.pass === true,
|
|
360
|
-
detail: judged.verdict?.summary && judged.verdict.summary.trim().length > 0
|
|
361
|
-
? judged.verdict.summary.trim()
|
|
362
|
-
: judged.error ?? "missing or invalid judge verdict",
|
|
363
|
-
});
|
|
364
|
-
}
|
|
365
|
-
const passedChecks = checks.filter((check) => check.ok).length;
|
|
366
|
-
return {
|
|
367
|
-
caseId: testCase.id,
|
|
368
|
-
question: testCase.question,
|
|
369
|
-
...(testCase.file ? { file: testCase.file } : {}),
|
|
370
|
-
ok: passedChecks === checks.length,
|
|
371
|
-
wordCount,
|
|
372
|
-
passedChecks,
|
|
373
|
-
totalChecks: checks.length,
|
|
374
|
-
checks,
|
|
375
|
-
};
|
|
376
|
-
}
|
|
377
|
-
function buildTestTargetResult(target, caseResults, options = {}) {
|
|
378
|
-
const passedCases = caseResults.filter((result) => result.ok).length;
|
|
379
|
-
const passedChecks = caseResults.reduce((total, result) => total + result.passedChecks, 0);
|
|
380
|
-
const totalChecks = caseResults.reduce((total, result) => total + result.totalChecks, 0);
|
|
381
|
-
return {
|
|
382
|
-
target: {
|
|
383
|
-
type: target.type,
|
|
384
|
-
name: target.name,
|
|
385
|
-
path: target.path,
|
|
386
|
-
workflow: target.workflow,
|
|
387
|
-
},
|
|
388
|
-
...(options.sandboxPath ? { sandbox_path: options.sandboxPath } : {}),
|
|
389
|
-
ok: passedCases === caseResults.length,
|
|
390
|
-
passedCases,
|
|
391
|
-
totalCases: caseResults.length,
|
|
392
|
-
passedChecks,
|
|
393
|
-
totalChecks,
|
|
394
|
-
caseResults,
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
function buildTestTargetRun(sourcePath, spec, results, executor, generatedAt) {
|
|
398
|
-
return {
|
|
399
|
-
kind: "interf-test-target-run",
|
|
400
|
-
version: 1,
|
|
401
|
-
generated_at: generatedAt ?? new Date().toISOString(),
|
|
402
|
-
spec: {
|
|
403
|
-
id: spec.id,
|
|
404
|
-
name: spec.name,
|
|
405
|
-
type: spec.type,
|
|
406
|
-
file: spec.filePath,
|
|
407
|
-
...(spec.description ? { description: spec.description } : {}),
|
|
408
|
-
case_count: spec.cases.length,
|
|
409
|
-
cases: spec.cases.map((testCase) => ({
|
|
410
|
-
id: testCase.id,
|
|
411
|
-
question: testCase.question,
|
|
412
|
-
...(testCase.file ? { file: testCase.file } : {}),
|
|
413
|
-
...(testCase.answer ? { answer: testCase.answer } : {}),
|
|
414
|
-
...(testCase.strictness ? { strictness: testCase.strictness } : {}),
|
|
415
|
-
...(testCase.expect ? { expect: testCase.expect } : {}),
|
|
416
|
-
})),
|
|
417
|
-
},
|
|
418
|
-
source_path: sourcePath,
|
|
419
|
-
...(executor ? { executor: buildRuntimeExecutorInfo(executor) } : {}),
|
|
420
|
-
target_count: results.length,
|
|
421
|
-
results,
|
|
422
|
-
};
|
|
423
|
-
}
|
|
424
|
-
export function runTargetTests(sourcePath, spec, targets) {
|
|
425
|
-
if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
|
|
426
|
-
throw new Error("This test needs a live executor. Use runTargetTestsWithJudge instead.");
|
|
427
|
-
}
|
|
428
|
-
for (const target of targets) {
|
|
429
|
-
if (target.type !== spec.type) {
|
|
430
|
-
throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
const results = targets.map((target) => buildTestTargetResult(target, spec.cases.map((testCase) => runTestCase(target, testCase))));
|
|
434
|
-
return buildTestTargetRun(sourcePath, spec, results);
|
|
435
|
-
}
|
|
436
|
-
export async function runTargetTestsWithJudge(sourcePath, spec, targets, executor, options = {}) {
|
|
437
|
-
const preserveMode = options.preserveSandboxes ?? "on-failure";
|
|
438
|
-
const artifactRootPath = options.artifactRootPath ?? sourcePath;
|
|
439
|
-
const generatedAt = new Date().toISOString();
|
|
440
|
-
const sandboxRunId = `${generatedAt.replace(/[:.]/g, "-")}-${spec.id}`;
|
|
441
|
-
for (const target of targets) {
|
|
442
|
-
if (target.type !== spec.type) {
|
|
443
|
-
throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
const results = [];
|
|
447
|
-
for (const [index, target] of targets.entries()) {
|
|
448
|
-
const sandbox = createTestSandbox(target);
|
|
449
|
-
try {
|
|
450
|
-
const sandboxTarget = {
|
|
451
|
-
...target,
|
|
452
|
-
path: sandbox.targetPath,
|
|
453
|
-
};
|
|
454
|
-
const caseResults = [];
|
|
455
|
-
for (const testCase of spec.cases) {
|
|
456
|
-
if (testCaseNeedsExecutor(testCase)) {
|
|
457
|
-
caseResults.push(await runLiveTestCase(sandboxTarget, testCase, executor));
|
|
458
|
-
}
|
|
459
|
-
else {
|
|
460
|
-
caseResults.push(await runTestCaseWithJudge(sandboxTarget, testCase, executor));
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
let sandboxPath;
|
|
464
|
-
const targetResult = buildTestTargetResult(target, caseResults);
|
|
465
|
-
const shouldPreserveSandbox = preserveMode === "always" || !targetResult.ok;
|
|
466
|
-
if (shouldPreserveSandbox) {
|
|
467
|
-
const sandboxRoot = targetTestSandboxesPath(artifactRootPath, target.type);
|
|
468
|
-
mkdirSync(sandboxRoot, { recursive: true });
|
|
469
|
-
const gitignorePath = targetTestSandboxGitignorePath(artifactRootPath, target.type);
|
|
470
|
-
if (!existsSync(gitignorePath)) {
|
|
471
|
-
writeFileSync(gitignorePath, "*\n!.gitignore\n");
|
|
472
|
-
}
|
|
473
|
-
const sandboxPathName = `${String(index + 1).padStart(2, "0")}-${normalizeTestId(target.name) || target.type}`;
|
|
474
|
-
sandbox.preserve(join(sandboxRoot, sandboxRunId, sandboxPathName));
|
|
475
|
-
sandboxPath = sandbox.targetPath;
|
|
476
|
-
}
|
|
477
|
-
results.push(buildTestTargetResult(target, caseResults, { sandboxPath }));
|
|
478
|
-
}
|
|
479
|
-
finally {
|
|
480
|
-
sandbox.cleanup();
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
return buildTestTargetRun(sourcePath, spec, results, executor, generatedAt);
|
|
484
|
-
}
|
|
485
|
-
export async function runTargetTestsAuto(sourcePath, spec, targets, options) {
|
|
486
|
-
if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
|
|
487
|
-
if (!options?.executor) {
|
|
488
|
-
throw new Error("This test needs a live local executor, but no executor was provided.");
|
|
489
|
-
}
|
|
490
|
-
return runTargetTestsWithJudge(sourcePath, spec, targets, options.executor, {
|
|
491
|
-
preserveSandboxes: options.preserveSandboxes,
|
|
492
|
-
artifactRootPath: options.artifactRootPath,
|
|
493
|
-
});
|
|
494
|
-
}
|
|
495
|
-
return runTargetTests(sourcePath, spec, targets);
|
|
496
|
-
}
|
|
497
|
-
export function saveTargetTestRun(artifactRootPath, result) {
|
|
498
|
-
const dirPath = targetTestRunsPath(artifactRootPath, result.spec.type);
|
|
499
|
-
mkdirSync(dirPath, { recursive: true });
|
|
500
|
-
const gitignorePath = targetTestRunGitignorePath(artifactRootPath, result.spec.type);
|
|
501
|
-
if (!existsSync(gitignorePath)) {
|
|
502
|
-
writeFileSync(gitignorePath, "*\n!.gitignore\n");
|
|
503
|
-
}
|
|
504
|
-
const timestamp = result.generated_at.replace(/[:.]/g, "-");
|
|
505
|
-
const runDirPath = join(dirPath, `${timestamp}-${result.spec.id}`);
|
|
506
|
-
mkdirSync(runDirPath, { recursive: true });
|
|
507
|
-
const manifestPath = join(runDirPath, "manifest.json");
|
|
508
|
-
writeFileSync(manifestPath, `${JSON.stringify({
|
|
509
|
-
kind: "interf-test-target-run-manifest",
|
|
510
|
-
version: 1,
|
|
511
|
-
generated_at: result.generated_at,
|
|
512
|
-
spec: {
|
|
513
|
-
id: result.spec.id,
|
|
514
|
-
name: result.spec.name,
|
|
515
|
-
type: result.spec.type,
|
|
516
|
-
case_count: result.spec.case_count,
|
|
517
|
-
},
|
|
518
|
-
result_file: "run.json",
|
|
519
|
-
target_count: result.target_count,
|
|
520
|
-
passed_targets: result.results.filter((entry) => entry.ok).length,
|
|
521
|
-
}, null, 2)}\n`);
|
|
522
|
-
const runPath = join(runDirPath, "run.json");
|
|
523
|
-
writeFileSync(runPath, `${JSON.stringify(result, null, 2)}\n`);
|
|
524
|
-
return runPath;
|
|
525
|
-
}
|
|
1
|
+
// Compatibility shim for historical `src/lib/test-execution.ts` imports.
|
|
2
|
+
// Source of truth: `src/packages/testing/test-execution.ts`.
|
|
3
|
+
export * from "../packages/testing/test-execution.js";
|