@vibe-agent-toolkit/agent-skills 0.1.39-rc.9 → 0.1.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files-config.d.ts +61 -0
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +262 -6
- package/dist/files-config.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/plugin-distribution-layout.d.ts +70 -0
- package/dist/plugin-distribution-layout.d.ts.map +1 -0
- package/dist/plugin-distribution-layout.js +97 -0
- package/dist/plugin-distribution-layout.js.map +1 -0
- package/dist/skill-packager.d.ts +43 -1
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +131 -53
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts +2 -2
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +33 -10
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +23 -8
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-source/sources/workspace-source.d.ts.map +1 -1
- package/dist/skill-source/sources/workspace-source.js +16 -8
- package/dist/skill-source/sources/workspace-source.js.map +1 -1
- package/dist/skill-source/stage.js +6 -2
- package/dist/skill-source/stage.js.map +1 -1
- package/dist/skill-test/configure-writer.d.ts +19 -11
- package/dist/skill-test/configure-writer.d.ts.map +1 -1
- package/dist/skill-test/configure-writer.js +23 -15
- package/dist/skill-test/configure-writer.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts +72 -0
- package/dist/skill-test/declared-env.d.ts.map +1 -0
- package/dist/skill-test/declared-env.js +85 -0
- package/dist/skill-test/declared-env.js.map +1 -0
- package/dist/skill-test/eval-inputs.d.ts +159 -0
- package/dist/skill-test/eval-inputs.d.ts.map +1 -0
- package/dist/skill-test/eval-inputs.js +158 -0
- package/dist/skill-test/eval-inputs.js.map +1 -0
- package/dist/skill-test/evals-template.d.ts.map +1 -1
- package/dist/skill-test/evals-template.js +27 -10
- package/dist/skill-test/evals-template.js.map +1 -1
- package/dist/skill-test/exit-codes.d.ts +43 -5
- package/dist/skill-test/exit-codes.d.ts.map +1 -1
- package/dist/skill-test/exit-codes.js +60 -9
- package/dist/skill-test/exit-codes.js.map +1 -1
- package/dist/skill-test/experimenter-prompt.d.ts +22 -0
- package/dist/skill-test/experimenter-prompt.d.ts.map +1 -1
- package/dist/skill-test/experimenter-prompt.js +38 -2
- package/dist/skill-test/experimenter-prompt.js.map +1 -1
- package/dist/skill-test/grading-adapter.d.ts +42 -0
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +63 -1
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +73 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +20 -5
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/harness-location.d.ts +22 -4
- package/dist/skill-test/harness-location.d.ts.map +1 -1
- package/dist/skill-test/harness-location.js +83 -18
- package/dist/skill-test/harness-location.js.map +1 -1
- package/dist/skill-test/index.d.ts +7 -4
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +7 -4
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +23 -0
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +34 -0
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +1 -2
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +193 -19
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +366 -76
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +31 -15
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/vendor-manifest.d.ts +3 -3
- package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
- package/dist/skill-test/vendor-manifest.js +13 -3
- package/dist/skill-test/vendor-manifest.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +134 -1
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +18 -14
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +26 -1
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +6 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness-location.js","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"harness-location.js","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEvF,sDAAsD;AACtD,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,gFAAgF;AAChF,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;AACjD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,UAAoB;IACnD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,oBAAoB,CAAC,sDAAsD,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtF,OAAO,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,kBAAkB,CAAC,UAAoB,EAAE,OAAgB;IACvE,MAAM,IAAI,GAAG,OAAO,IAAI,gBAAgB,EAAE,CAAC;IAC3C,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAe;IAC5D,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpD,iHAAiH;QACjH,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,oBAAoB,CAAC,qDAAqD,OAAO,2BAA2B,CAAC,CAAC;QAC1H,CAAC;QACD,iHAAiH;QACjH,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,oBAAoB,CAAC,oDAAoD,OAAO,2BAA2B,CAAC,CAAC;QACzH,CAAC;QACD,QAAQ,GAAG,OAAO,CAAC;QACnB,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,mGAAmG;IACnG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO;IAE7B,mGAAmG;IACnG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAAC,6CAA6C,GAAG,GAAG,CAAC,CAAC;IACtF,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,mGAAmG;QACnG,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;QACxC,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,mGAAmG;YACnG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;AACH,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe,CAAC,IAAY,EAAE,KAAa;IAClD,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC3C,OAAO,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;AACtF,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,WAAmB;IACxD,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,OAAO,OAAO,KAAK,WAAW,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,MAAM,KAAK,OAAO;YAAE,MAAM,CAAC,0CAA0C;QACzE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,SAAS,mBAAmB,CAAC,SAAiB,EAAE,UAAkB;IAChE,6GAA6G;IAC7G,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO;IACnC,6GAA6G;IAC7G,MAAM,EAAE,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAAC,uDAAuD,SAAS,GAAG,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,EAAE,CAAC,GAAG,KAAK,QAAQ,IAAI,EAAE,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QACxF,MAAM,IAAI,oBAAoB,CAC5B,0BAA0B,SAAS,0CAA0C,EAAE,CAAC,GAAG,OAAO,UAAU,IAAI,CACzG,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,qBAAqB,CACnC,GAAW,EACX,UAAkB,EAClB,cAAsB,gBAAgB,EAAE;IAExC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACvC,8NAA8N;IAC9N,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAE3C,KAAK,MAAM,SAAS,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC;QACxD,mBAAmB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,8EAA8E;IAC9E,uEAAuE;IACvE,mGAAmG;IACnG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,CAAC,2CAA2C;IAC9E,mGAAmG;IACnG,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;IAC7C,IAAI,IAAI,KAAK,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACnD,MAAM,IAAI,oBAAoB,CAAC,gBAAgB,QAAQ,wBAAwB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvG,CAAC;AACH,CAAC"}
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
export { EvalEntrySchema, EvalInputError, EvalSuiteSchema, parseEvalSuite, stageEvalWorkspaces, type EvalEntry, type EvalSuite, type StageEvalWorkspacesInput, } from './eval-inputs.js';
|
|
1
2
|
export { FrictionCategorySchema, FrictionItemSchema, FrictionReportJsonSchema, FrictionReportSchema, FrictionSeveritySchema, type FrictionItem, type FrictionReport, } from './friction-schema.js';
|
|
2
|
-
export { GradingSkewError, parseGradingJson, type NormalizedGrading } from './grading-adapter.js';
|
|
3
|
+
export { assertGradingNonce, GradingNonceError, GradingSkewError, parseGradingJson, reconcileGrading, type GradingVerdict, type NormalizedGrading, } from './grading-adapter.js';
|
|
3
4
|
export { GradedExpectationSchema, GradingReportJsonSchema, GradingReportSchema, GradingSummarySchema, type GradedExpectation, type GradingReport, type GradingSummary, } from './grading-schema.js';
|
|
4
5
|
export { assertSafeHarnessRoot, assertSafeWorkdir, deriveHarnessKey, HarnessLocationError, resolveHarnessRoot, } from './harness-location.js';
|
|
5
6
|
export { computeReconcilePlan, StagedEntrySchema, StagedManifestSchema, type ReconcilePlan, type StagedEntry, type StagedManifest, } from './manifest.js';
|
|
6
|
-
export {
|
|
7
|
+
export { assembleChildEnv, computeEnvTokens, interpolateEnvValue, resolveInjectEnv, UnknownEnvTokenError, type AssembleChildEnvInput, type AssembledChildEnv, type EnvInterpolationTokens, type EnvTokenInputs, } from './declared-env.js';
|
|
8
|
+
export { BootstrapNeededError, InternalHarnessError, mapErrorToExitCode, SecurityAckError, SkillBuildError, SkillTestExitCode, type SkillTestExitCodeValue, } from './exit-codes.js';
|
|
9
|
+
export { BuildHookError, runPreStageBuild } from './build-hook.js';
|
|
7
10
|
export { runPreflight, type PreflightCheck, type PreflightInput, type PreflightResult, } from './preflight.js';
|
|
8
|
-
export { assertPromptInvariants, buildExperimenterPrompt, DEFAULT_EXPERIMENTER_PROMPT, PromptInvariantError, type BuildPromptOptions, } from './experimenter-prompt.js';
|
|
11
|
+
export { appendIntegrityNonceDirective, assertPromptInvariants, buildExperimenterPrompt, DEFAULT_EXPERIMENTER_PROMPT, PromptInvariantError, redactNonce, REDACTED_NONCE_PLACEHOLDER, type BuildPromptOptions, } from './experimenter-prompt.js';
|
|
9
12
|
export { acquireHarnessLock, HarnessLockBusyError, type HarnessLock } from './lock.js';
|
|
10
13
|
export { computeDirContentHash, descriptorToSource, stagedDirName, stageHarness, type StageHarnessOptions, type StageHarnessResult, type StageItem, } from './staging.js';
|
|
11
14
|
export { regenerateVendoredManifest, verifyVendoredManifest, VendoredManifestSchema } from './vendor-manifest.js';
|
|
12
15
|
export { upsertTestConfig } from './configure-writer.js';
|
|
13
16
|
export { buildEvalsTemplate, writeEvalsTemplate } from './evals-template.js';
|
|
14
|
-
export { runSkillTestHarness, type RunHarnessOptions, type RunHarnessResult, } from './run-harness.js';
|
|
17
|
+
export { buildDryRunSummary, isAcknowledged, runSkillTestHarness, SKILL_TEST_BUILTIN_CAPS, verdictExitCode, type DryRunSummaryInput, type RunHarnessOptions, type RunHarnessResult, } from './run-harness.js';
|
|
15
18
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/skill-test/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,wBAAwB,EACxB,oBAAoB,EACpB,sBAAsB,EACtB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/skill-test/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,cAAc,EACd,eAAe,EACf,cAAc,EACd,mBAAmB,EACnB,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,wBAAwB,GAC9B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,wBAAwB,EACxB,oBAAoB,EACpB,sBAAsB,EACtB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,KAAK,cAAc,EACnB,KAAK,iBAAiB,GACvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,oBAAoB,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,cAAc,GACpB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,cAAc,GACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,KAAK,sBAAsB,GAC5B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,YAAY,EACZ,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,2BAA2B,EAC3B,oBAAoB,EACpB,WAAW,EACX,0BAA0B,EAC1B,KAAK,kBAAkB,GACxB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AACvF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,SAAS,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAClH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,GACtB,MAAM,kBAAkB,CAAC"}
|
package/dist/skill-test/index.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
export { EvalEntrySchema, EvalInputError, EvalSuiteSchema, parseEvalSuite, stageEvalWorkspaces, } from './eval-inputs.js';
|
|
1
2
|
export { FrictionCategorySchema, FrictionItemSchema, FrictionReportJsonSchema, FrictionReportSchema, FrictionSeveritySchema, } from './friction-schema.js';
|
|
2
|
-
export { GradingSkewError, parseGradingJson } from './grading-adapter.js';
|
|
3
|
+
export { assertGradingNonce, GradingNonceError, GradingSkewError, parseGradingJson, reconcileGrading, } from './grading-adapter.js';
|
|
3
4
|
export { GradedExpectationSchema, GradingReportJsonSchema, GradingReportSchema, GradingSummarySchema, } from './grading-schema.js';
|
|
4
5
|
export { assertSafeHarnessRoot, assertSafeWorkdir, deriveHarnessKey, HarnessLocationError, resolveHarnessRoot, } from './harness-location.js';
|
|
5
6
|
export { computeReconcilePlan, StagedEntrySchema, StagedManifestSchema, } from './manifest.js';
|
|
6
|
-
export {
|
|
7
|
+
export { assembleChildEnv, computeEnvTokens, interpolateEnvValue, resolveInjectEnv, UnknownEnvTokenError, } from './declared-env.js';
|
|
8
|
+
export { BootstrapNeededError, InternalHarnessError, mapErrorToExitCode, SecurityAckError, SkillBuildError, SkillTestExitCode, } from './exit-codes.js';
|
|
9
|
+
export { BuildHookError, runPreStageBuild } from './build-hook.js';
|
|
7
10
|
export { runPreflight, } from './preflight.js';
|
|
8
|
-
export { assertPromptInvariants, buildExperimenterPrompt, DEFAULT_EXPERIMENTER_PROMPT, PromptInvariantError, } from './experimenter-prompt.js';
|
|
11
|
+
export { appendIntegrityNonceDirective, assertPromptInvariants, buildExperimenterPrompt, DEFAULT_EXPERIMENTER_PROMPT, PromptInvariantError, redactNonce, REDACTED_NONCE_PLACEHOLDER, } from './experimenter-prompt.js';
|
|
9
12
|
export { acquireHarnessLock, HarnessLockBusyError } from './lock.js';
|
|
10
13
|
export { computeDirContentHash, descriptorToSource, stagedDirName, stageHarness, } from './staging.js';
|
|
11
14
|
export { regenerateVendoredManifest, verifyVendoredManifest, VendoredManifestSchema } from './vendor-manifest.js';
|
|
12
15
|
export { upsertTestConfig } from './configure-writer.js';
|
|
13
16
|
export { buildEvalsTemplate, writeEvalsTemplate } from './evals-template.js';
|
|
14
|
-
export { runSkillTestHarness, } from './run-harness.js';
|
|
17
|
+
export { buildDryRunSummary, isAcknowledged, runSkillTestHarness, SKILL_TEST_BUILTIN_CAPS, verdictExitCode, } from './run-harness.js';
|
|
15
18
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/skill-test/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,wBAAwB,EACxB,oBAAoB,EACpB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/skill-test/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,cAAc,EACd,eAAe,EACf,cAAc,EACd,mBAAmB,GAIpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,wBAAwB,EACxB,oBAAoB,EACpB,sBAAsB,GAGvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,GAGjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,GAIrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,oBAAoB,GAIrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,GAKrB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EACf,iBAAiB,GAElB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EACL,YAAY,GAIb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,6BAA6B,EAC7B,sBAAsB,EACtB,uBAAuB,EACvB,2BAA2B,EAC3B,oBAAoB,EACpB,WAAW,EACX,0BAA0B,GAE3B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAoB,MAAM,WAAW,CAAC;AACvF,OAAO,EACL,qBAAqB,EACrB,kBAAkB,EAClB,aAAa,EACb,YAAY,GAIb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAClH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,GAIhB,MAAM,kBAAkB,CAAC"}
|
|
@@ -13,4 +13,27 @@ export interface HarnessLock {
|
|
|
13
13
|
export declare function acquireHarnessLock(harnessRoot: string, opts?: {
|
|
14
14
|
wait?: boolean;
|
|
15
15
|
}): HarnessLock;
|
|
16
|
+
export interface InstallSignalCleanupOptions {
|
|
17
|
+
/**
|
|
18
|
+
* Invoked once on the first trapped signal, BEFORE exiting — the place to
|
|
19
|
+
* release the harness lock and remove the harness dir so a Ctrl-C mid-run does
|
|
20
|
+
* not leave `.vat-skill-test.lock` (or staged untrusted bytes) behind.
|
|
21
|
+
*/
|
|
22
|
+
onSignal: () => void;
|
|
23
|
+
/**
|
|
24
|
+
* Exit hook, injectable for tests. Defaults to `process.exit`. Called with the
|
|
25
|
+
* conventional 128+signal code so the signal is honored, not swallowed.
|
|
26
|
+
*/
|
|
27
|
+
exit?: (code: number) => void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Trap SIGINT/SIGTERM while the harness lock is held. On the first such signal
|
|
31
|
+
* the handler removes ITSELF (no listener leak, no re-entry), runs `onSignal`
|
|
32
|
+
* (release lock + clean up), then exits with the conventional 128+signal code.
|
|
33
|
+
*
|
|
34
|
+
* Returns a disposer that removes the handlers — the caller MUST call it on
|
|
35
|
+
* normal completion so listeners do not accumulate across runs in a long-lived
|
|
36
|
+
* process (and so tests stay isolated).
|
|
37
|
+
*/
|
|
38
|
+
export declare function installSignalCleanup(opts: InstallSignalCleanupOptions): () => void;
|
|
16
39
|
//# sourceMappingURL=lock.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../src/skill-test/lock.ts"],"names":[],"mappings":"AAIA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,QAAQ,EAAE,MAAM;CAI7B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,WAAW,CAwBlG"}
|
|
1
|
+
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../src/skill-test/lock.ts"],"names":[],"mappings":"AAIA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,QAAQ,EAAE,MAAM;CAI7B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,WAAW,CAwBlG;AAWD,MAAM,WAAW,2BAA2B;IAC1C;;;;OAIG;IACH,QAAQ,EAAE,MAAM,IAAI,CAAC;IACrB;;;OAGG;IACH,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC/B;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,2BAA2B,GAAG,MAAM,IAAI,CAgBlF"}
|
package/dist/skill-test/lock.js
CHANGED
|
@@ -39,4 +39,38 @@ export function acquireHarnessLock(harnessRoot, opts = {}) {
|
|
|
39
39
|
},
|
|
40
40
|
};
|
|
41
41
|
}
|
|
42
|
+
/** Conventional shell exit-code base for a process terminated by a signal. */
|
|
43
|
+
const SIGNAL_EXIT_BASE = 128;
|
|
44
|
+
/** Signals we trap so an interrupted run cleans up instead of leaking the lock. */
|
|
45
|
+
const SIGNAL_NUMBERS = [
|
|
46
|
+
['SIGINT', 2],
|
|
47
|
+
['SIGTERM', 15],
|
|
48
|
+
];
|
|
49
|
+
/**
|
|
50
|
+
* Trap SIGINT/SIGTERM while the harness lock is held. On the first such signal
|
|
51
|
+
* the handler removes ITSELF (no listener leak, no re-entry), runs `onSignal`
|
|
52
|
+
* (release lock + clean up), then exits with the conventional 128+signal code.
|
|
53
|
+
*
|
|
54
|
+
* Returns a disposer that removes the handlers — the caller MUST call it on
|
|
55
|
+
* normal completion so listeners do not accumulate across runs in a long-lived
|
|
56
|
+
* process (and so tests stay isolated).
|
|
57
|
+
*/
|
|
58
|
+
export function installSignalCleanup(opts) {
|
|
59
|
+
const exit = opts.exit ?? ((code) => { process.exit(code); });
|
|
60
|
+
const registered = [];
|
|
61
|
+
const remove = () => {
|
|
62
|
+
for (const [sig, handler] of registered)
|
|
63
|
+
process.off(sig, handler);
|
|
64
|
+
};
|
|
65
|
+
for (const [sig, num] of SIGNAL_NUMBERS) {
|
|
66
|
+
const handler = () => {
|
|
67
|
+
remove();
|
|
68
|
+
opts.onSignal();
|
|
69
|
+
exit(SIGNAL_EXIT_BASE + num);
|
|
70
|
+
};
|
|
71
|
+
process.on(sig, handler);
|
|
72
|
+
registered.push([sig, handler]);
|
|
73
|
+
}
|
|
74
|
+
return remove;
|
|
75
|
+
}
|
|
42
76
|
//# sourceMappingURL=lock.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lock.js","sourceRoot":"","sources":["../../src/skill-test/lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,QAAgB;QAC1B,KAAK,CAAC,sDAAsD,QAAQ,yDAAyD,CAAC,CAAC;QAC/H,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAMD;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAmB,EAAE,OAA2B,EAAE;IACnF,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC7E,wGAAwG;IACxG,KAAK,IAAI,CAAC,IAAI,CAAC;IACf,IAAI,EAAU,CAAC;IACf,IAAI,CAAC;QACH,8EAA8E;QAC9E,mGAAmG;QACnG,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,SAAS,CAAC,EAAE,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,OAAO;QACL,OAAO;YACL,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"lock.js","sourceRoot":"","sources":["../../src/skill-test/lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,QAAgB;QAC1B,KAAK,CAAC,sDAAsD,QAAQ,yDAAyD,CAAC,CAAC;QAC/H,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAMD;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAmB,EAAE,OAA2B,EAAE;IACnF,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC7E,wGAAwG;IACxG,KAAK,IAAI,CAAC,IAAI,CAAC;IACf,IAAI,EAAU,CAAC;IACf,IAAI,CAAC;QACH,8EAA8E;QAC9E,mGAAmG;QACnG,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,SAAS,CAAC,EAAE,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,OAAO;QACL,OAAO;YACL,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,mFAAmF;AACnF,MAAM,cAAc,GAAqD;IACvE,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,SAAS,EAAE,EAAE,CAAC;CAChB,CAAC;AAgBF;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAiC;IACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAY,EAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAiD,EAAE,CAAC;IACpE,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrE,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzB,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../src/skill-test/preflight.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,kBAAkB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACxC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IAC1C,SAAS,EAAE,eAAe,CAAC;IAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,OAAO,CAAC;IAC3B,YAAY,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACzG,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,CAAC,EAAE,aAAa,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;CACnC;AAeD,wBAAgB,YAAY,CAAC,KAAK,EAAE,cAAc,GAAG,eAAe,
|
|
1
|
+
{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../src/skill-test/preflight.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,QAAQ,EACb,KAAK,eAAe,EACpB,KAAK,YAAY,EAClB,MAAM,2BAA2B,CAAC;AAEnC,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,kBAAkB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACxC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IAC1C,SAAS,EAAE,eAAe,CAAC;IAC3B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,OAAO,CAAC;IAC3B,YAAY,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACzG,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,CAAC,EAAE,aAAa,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;CACnC;AAeD,wBAAgB,YAAY,CAAC,KAAK,EAAE,cAAc,GAAG,eAAe,CA4DnE"}
|
|
@@ -30,8 +30,7 @@ export function runPreflight(input) {
|
|
|
30
30
|
}
|
|
31
31
|
checks.push(input.integrityOk()
|
|
32
32
|
? { name: 'vendored skill-creator integrity', passed: true, message: 'manifest verified' }
|
|
33
|
-
: { name: 'vendored skill-creator integrity', passed: false, message: 'hash manifest mismatch', suggestion: 'Re-sync the vendored copy.' });
|
|
34
|
-
checks.push(checkExists('eval input files', input.evalInputPaths), checkExists('declared dependencies', input.declaredDepDirs));
|
|
33
|
+
: { name: 'vendored skill-creator integrity', passed: false, message: 'hash manifest mismatch', suggestion: 'Re-sync the vendored copy.' }, checkExists('eval input files', input.evalInputPaths), checkExists('declared dependencies', input.declaredDepDirs));
|
|
35
34
|
let resolvedAuth = null;
|
|
36
35
|
try {
|
|
37
36
|
resolvedAuth = resolveAuth({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"preflight.js","sourceRoot":"","sources":["../../src/skill-test/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAErC,OAAO,EACL,kBAAkB,EAClB,WAAW,GAKZ,MAAM,2BAA2B,CAAC;AA4BnC,MAAM,cAAc,GAAG;IACrB,cAAc,EAAE,mBAAmB,EAAE,iBAAiB;IACtD,mBAAmB,EAAE,aAAa,EAAE,kBAAkB;CAC9C,CAAC;AAEX,SAAS,WAAW,CAAC,KAAa,EAAE,KAAe;IACjD,sGAAsG;IACtG,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;QACzB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,KAAK,CAAC,MAAM,UAAU,EAAE;QACvE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,yCAAyC,EAAE,CAAC;AACvI,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAqB;IAChD,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,kBAAkB,EAAE,CAAC;IAC3C,MAAM,CAAC,IAAI,CAAC,OAAO;QACjB,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE;QAC3D,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAEhH,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,QAAQ,IAAI,EAAE;YACpB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,8BAA8B;YAC1D,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,EAAE;gBAC/B,CAAC,CAAC,EAAE,UAAU,EAAE,iFAAiF,EAAE;gBACnG,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,IAAI,
|
|
1
|
+
{"version":3,"file":"preflight.js","sourceRoot":"","sources":["../../src/skill-test/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAErC,OAAO,EACL,kBAAkB,EAClB,WAAW,GAKZ,MAAM,2BAA2B,CAAC;AA4BnC,MAAM,cAAc,GAAG;IACrB,cAAc,EAAE,mBAAmB,EAAE,iBAAiB;IACtD,mBAAmB,EAAE,aAAa,EAAE,kBAAkB;CAC9C,CAAC;AAEX,SAAS,WAAW,CAAC,KAAa,EAAE,KAAe;IACjD,sGAAsG;IACtG,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC;QACzB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,KAAK,CAAC,MAAM,UAAU,EAAE;QACvE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,yCAAyC,EAAE,CAAC;AACvI,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAqB;IAChD,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,kBAAkB,EAAE,CAAC;IAC3C,MAAM,CAAC,IAAI,CAAC,OAAO;QACjB,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE;QAC3D,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAEhH,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,QAAQ,IAAI,EAAE;YACpB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,8BAA8B;YAC1D,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,EAAE;gBAC/B,CAAC,CAAC,EAAE,UAAU,EAAE,iFAAiF,EAAE;gBACnG,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,WAAW,EAAE;QACjB,CAAC,CAAC,EAAE,IAAI,EAAE,kCAAkC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,EAAE;QAC1F,CAAC,CAAC,EAAE,IAAI,EAAE,kCAAkC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,wBAAwB,EAAE,UAAU,EAAE,4BAA4B,EAAE,EAC5I,WAAW,CAAC,kBAAkB,EAAE,KAAK,CAAC,cAAc,CAAC,EACrD,WAAW,CAAC,uBAAuB,EAAE,KAAK,CAAC,eAAe,CAAC,CAC5D,CAAC;IAEF,IAAI,YAAY,GAAwB,IAAI,CAAC;IAC7C,IAAI,CAAC;QACH,YAAY,GAAG,WAAW,CAAC;YACzB,IAAI,EAAE,KAAK,CAAC,QAAQ;YACpB,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,KAAK,EAAE,KAAK,CAAC,SAAS;SACvB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,kBAAkB,YAAY,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,MAAM,cAAc,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,MAAM,WAAW,GAAG,aAAa,YAAY,CAAC,kBAAkB,GAAG,UAAU,GAAG,cAAc,EAAE,CAAC;QACjG,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,WAAW;SACrB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,qDAAqD,EAAE,CAAC,CAAC;IAChI,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC,YAAY,CAAC;IACrF,MAAM,eAAe,GAAG,SAAS,GAAG,cAAc,GAAG,YAAY,CAAC;IAClE,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,aAAa,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,MAAM,WAAW,GAAG,GAAG,SAAS,YAAY,cAAc,aAAa,YAAY,WAAW,eAAe,cAAc,UAAU,EAAE,CAAC;IACxI,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,eAAe;QACrB,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,WAAW;KACrB,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;AACvE,CAAC"}
|
|
@@ -8,19 +8,15 @@
|
|
|
8
8
|
* → build effective prompt + assertPromptInvariants → dry-run short-circuit
|
|
9
9
|
* → spawnHeadlessClaude → parseGradingJson → release lock → return result
|
|
10
10
|
*/
|
|
11
|
+
import type { SkillSourceDescriptor } from '@vibe-agent-toolkit/resources';
|
|
12
|
+
import type { ResolveSkillSourceContext, SkillSource } from '../skill-source/types.js';
|
|
13
|
+
import { type SkillTestExitCodeValue } from './exit-codes.js';
|
|
14
|
+
import { type PreflightInput } from './preflight.js';
|
|
15
|
+
import { type StageItem } from './staging.js';
|
|
11
16
|
/** Auth modes supported by the harness. */
|
|
12
17
|
export type HarnessAuthMode = 'inherit' | 'subscription' | 'api-key' | 'auto';
|
|
13
18
|
/** Auth mechanism requirements. */
|
|
14
19
|
export type HarnessAuthMechanism = 'subscription' | 'api-key';
|
|
15
|
-
/** A skill source descriptor (mirrors SkillSource union). */
|
|
16
|
-
export interface SkillSourceSpec {
|
|
17
|
-
path?: string;
|
|
18
|
-
npm?: string;
|
|
19
|
-
url?: string;
|
|
20
|
-
sha256?: string;
|
|
21
|
-
workspace?: string;
|
|
22
|
-
vendored?: true;
|
|
23
|
-
}
|
|
24
20
|
export interface RunHarnessOptions {
|
|
25
21
|
/** Primary skill names (subject set, required). */
|
|
26
22
|
skills: string[];
|
|
@@ -40,9 +36,9 @@ export interface RunHarnessOptions {
|
|
|
40
36
|
* Optional additional skill sources mapped by name. If provided, overrides
|
|
41
37
|
* the default path-based resolution for that skill name.
|
|
42
38
|
*/
|
|
43
|
-
withSources?: Record<string,
|
|
39
|
+
withSources?: Record<string, SkillSourceDescriptor>;
|
|
44
40
|
/** Additional optional skills to inject (with-optional). */
|
|
45
|
-
withOptional?: Record<string,
|
|
41
|
+
withOptional?: Record<string, SkillSourceDescriptor>;
|
|
46
42
|
/** Override the harness working directory (base for harnessKey derivation). */
|
|
47
43
|
workdir?: string;
|
|
48
44
|
/** Override the harness output directory (explicit full path). */
|
|
@@ -73,23 +69,201 @@ export interface RunHarnessOptions {
|
|
|
73
69
|
/** Stall watchdog in seconds. */
|
|
74
70
|
stall?: number;
|
|
75
71
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* Absent →
|
|
72
|
+
* Pre-resolved source for the subject (skills[0]); set by run.ts after
|
|
73
|
+
* project-aware resolution + build. A built dist dir for a declared skill, or
|
|
74
|
+
* the resolved as-is source. Absent → legacy { path: skills[0] }.
|
|
75
|
+
*/
|
|
76
|
+
subjectSource?: SkillSource;
|
|
77
|
+
/**
|
|
78
|
+
* Authored source dir for the subject, where the eval suite (`evals/`, incl.
|
|
79
|
+
* `fixtures/`) is maintained. Used to (a) overlay that suite onto a built/dist
|
|
80
|
+
* subject that doesn't carry it, and (b) write the bootstrap template when no
|
|
81
|
+
* suite exists yet. For a built declared skill this is the SOURCE skill dir,
|
|
82
|
+
* not the dist. Absent → derived from skills[0] (legacy).
|
|
83
|
+
*/
|
|
84
|
+
subjectScaffoldDir?: string;
|
|
85
|
+
/**
|
|
86
|
+
* True when run.ts actually rebuilt the subject (declared skill, no
|
|
87
|
+
* --no-build/--dry-run). Recorded in provenance. Absent/false → staged as-is.
|
|
79
88
|
*/
|
|
80
|
-
|
|
89
|
+
rebuilt?: boolean;
|
|
81
90
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
91
|
+
* True when the resolved reference is `buildable` — a real run WOULD build +
|
|
92
|
+
* stage it before spawning. False/absent for plain `source` subjects. Set by
|
|
93
|
+
* run.ts after project-aware resolution; used in the dry-run summary.
|
|
85
94
|
*/
|
|
86
|
-
|
|
95
|
+
wouldBuild?: boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Meaningful only when wouldBuild is true and dryRun is true. True = the
|
|
98
|
+
* dry-run staged the EXISTING on-disk dist without rebuilding (may be stale).
|
|
99
|
+
* False = no dist existed yet so the preview fell back to the source dir.
|
|
100
|
+
* Absent when not a dry-run or when the subject is a plain source.
|
|
101
|
+
*/
|
|
102
|
+
dryRunStagedExistingDist?: boolean;
|
|
103
|
+
/** Feature B: explicit env var injections (interpolated at stage time). */
|
|
104
|
+
env?: Record<string, string>;
|
|
105
|
+
/** Feature A: host env var names to forward to the experimenter if present. */
|
|
106
|
+
passEnv?: readonly string[];
|
|
107
|
+
/**
|
|
108
|
+
* Opt-in eval gating. When true and any eval fails, the run returns exit
|
|
109
|
+
* EvalFailure (4) instead of Ok. Default (false/absent): a completed run
|
|
110
|
+
* always returns Ok and the pass/fail count lives only in the summary.
|
|
111
|
+
*/
|
|
112
|
+
failOnEvalFailure?: boolean;
|
|
87
113
|
}
|
|
88
114
|
export interface RunHarnessResult {
|
|
89
115
|
harnessPath: string;
|
|
90
116
|
exitCode: number;
|
|
91
117
|
summary: string;
|
|
92
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Built-in cost/runtime safety ceilings for the experimenter spawn. These are the
|
|
121
|
+
* SAME values the harness applies as defaults, but exported as an explicit cap so
|
|
122
|
+
* the CLI precedence layer (run.ts) can enforce a critical asymmetry:
|
|
123
|
+
*
|
|
124
|
+
* - a CLI flag (explicit operator intent, typed at the terminal for THIS run)
|
|
125
|
+
* may RAISE a knob above the built-in ceiling;
|
|
126
|
+
* - a value sourced from a committed `test.*` config (which rides along in an
|
|
127
|
+
* untrusted subject repo you may only be testing) may only LOWER a ceiling,
|
|
128
|
+
* never raise it — so cloning + testing a hostile skill can't silently
|
|
129
|
+
* escalate the $5 / 50-turn / 5-minute budget the run bills against.
|
|
130
|
+
*
|
|
131
|
+
* `timeoutSeconds` is expressed in seconds to match the `--timeout`/`test.timeout`
|
|
132
|
+
* unit (the harness multiplies by 1000 internally; see resolveTimeoutMs).
|
|
133
|
+
*/
|
|
134
|
+
export declare const SKILL_TEST_BUILTIN_CAPS: {
|
|
135
|
+
readonly maxTurns: 50;
|
|
136
|
+
readonly maxBudgetUsd: 5;
|
|
137
|
+
readonly timeoutSeconds: number;
|
|
138
|
+
};
|
|
139
|
+
export declare function resolveTimeoutMs(opts: RunHarnessOptions): number;
|
|
140
|
+
/**
|
|
141
|
+
* Map an eval verdict to a process exit code. Default behavior: a completed run
|
|
142
|
+
* exits Ok regardless of pass/fail (the count lives in the summary). When
|
|
143
|
+
* `failOnEvalFailure` is set, a failing verdict escalates to EvalFailure (4) so
|
|
144
|
+
* CI can gate on eval outcomes without conflating them with harness breakage.
|
|
145
|
+
*/
|
|
146
|
+
export declare function verdictExitCode(allPassed: boolean, failOnEvalFailure: boolean): SkillTestExitCodeValue;
|
|
147
|
+
export declare function resolveStallMs(opts: RunHarnessOptions): number | undefined;
|
|
148
|
+
export declare function resolveKnobs(opts: RunHarnessOptions): {
|
|
149
|
+
model?: string;
|
|
150
|
+
maxTurns: number;
|
|
151
|
+
maxBudgetUsd: number;
|
|
152
|
+
stallMs?: number;
|
|
153
|
+
};
|
|
154
|
+
/**
|
|
155
|
+
* Detect the plugin-root layout for a `{ path }` source. The resolver later COPIES
|
|
156
|
+
* the source into a temp dir (losing its plugin ancestry), so we must detect here
|
|
157
|
+
* — while the true on-disk source dir is still known — by resolving the path spec
|
|
158
|
+
* against repoRoot and walking up for `.claude-plugin/plugin.json`. Non-`{path}`
|
|
159
|
+
* sources (npm/url/vendored/workspace) have no local source tree to walk, so they
|
|
160
|
+
* are always staged flat (returns undefined). undefined → flat staging.
|
|
161
|
+
*/
|
|
162
|
+
export declare function detectItemPluginLayout(source: SkillSource, repoRoot: string): StageItem['pluginLayout'] | undefined;
|
|
163
|
+
export declare function makeStageItem(name: string, source: SkillSource, repoRoot: string, role: 'subject' | undefined): StageItem;
|
|
164
|
+
export declare function buildStageItems(opts: RunHarnessOptions, repoRoot: string): StageItem[];
|
|
165
|
+
export declare function buildResolveCtx(harnessRoot: string, repoRoot: string): ResolveSkillSourceContext;
|
|
166
|
+
/**
|
|
167
|
+
* Return a sensible dummy value for a given CLI flag so that value-validation
|
|
168
|
+
* doesn't reject the argument before `--help` can short-circuit the session.
|
|
169
|
+
*/
|
|
170
|
+
export declare function flagDummyValueFor(flag: string): string;
|
|
171
|
+
export declare function buildPreflightInput(evalsPath: string, pluginDirs: string[], opts: RunHarnessOptions, knobs: {
|
|
172
|
+
maxBudgetUsd: number;
|
|
173
|
+
}): PreflightInput;
|
|
174
|
+
export declare function renderPreflightSummary(checks: {
|
|
175
|
+
name: string;
|
|
176
|
+
passed: boolean;
|
|
177
|
+
message: string;
|
|
178
|
+
}[]): string;
|
|
179
|
+
/**
|
|
180
|
+
* The single source of truth for "is this run acknowledged?": a dry-run never
|
|
181
|
+
* executes skill code (so it is implicitly acknowledged), otherwise the caller
|
|
182
|
+
* must pass --i-understand-this-runs-skill-code. Narrowed to the two fields it
|
|
183
|
+
* reads so run.ts can reuse the SAME predicate to gate the pre-build ack check
|
|
184
|
+
* (the harness Step-6 check and the run.ts pre-build check cannot diverge).
|
|
185
|
+
*/
|
|
186
|
+
export declare function isAcknowledged(opts: Pick<RunHarnessOptions, 'dryRun' | 'acknowledgedRunsSkillCode'>): boolean;
|
|
187
|
+
/**
|
|
188
|
+
* Translate a non-success spawn outcome into an InternalHarnessError (exit 1).
|
|
189
|
+
* A stall, a timeout, OR a non-zero exit are each authoritative — a non-zero exit
|
|
190
|
+
* is never laundered into a PASS even if a grading.json happens to be on disk.
|
|
191
|
+
*/
|
|
192
|
+
export declare function assertExperimenterSucceeded(spawnResult: {
|
|
193
|
+
stalled: boolean;
|
|
194
|
+
timedOut: boolean;
|
|
195
|
+
status: number;
|
|
196
|
+
}, stallMs: number | undefined, timeoutMs: number): void;
|
|
197
|
+
/**
|
|
198
|
+
* Resolve the PERSISTENT location where a bootstrap scaffold should be written
|
|
199
|
+
* so "fill it in and re-run" actually works for the user (the staged copy is
|
|
200
|
+
* ephemeral). When the subject is a local `{path}` source we scaffold next to
|
|
201
|
+
* that real source dir; otherwise we anchor under the repo root by skill name.
|
|
202
|
+
*/
|
|
203
|
+
/** The subject skill's display name (trailing segment of the positional skill arg). */
|
|
204
|
+
export declare function subjectSkillName(opts: RunHarnessOptions): string;
|
|
205
|
+
export declare function resolveScaffoldEvalsPath(opts: RunHarnessOptions, repoRoot: string, evalsSubpath: string): string;
|
|
206
|
+
/** Parse the staged eval suite and materialize each eval's input `files` into
|
|
207
|
+
* `<harnessRoot>/workspaces/<id>/`. Returns the workspaces root. The dir is wiped
|
|
208
|
+
* first so a reused harness root cannot leak a prior run's inputs. Throws
|
|
209
|
+
* {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
|
|
210
|
+
* missing input file. */
|
|
211
|
+
export declare function stageWorkspacesForRun(evalsPath: string, harnessRoot: string): string;
|
|
212
|
+
/** Inputs for the dry-run summary string. */
|
|
213
|
+
export interface DryRunSummaryInput {
|
|
214
|
+
/** True when the resolved subject is buildable (a real run would build + stage it). */
|
|
215
|
+
wouldBuild: boolean;
|
|
216
|
+
/**
|
|
217
|
+
* When wouldBuild is true: true = the dry-run staged the existing dist without
|
|
218
|
+
* rebuilding (may be stale); false = no dist existed, fell back to source dir.
|
|
219
|
+
*/
|
|
220
|
+
dryRunStagedExistingDist?: boolean;
|
|
221
|
+
/** Absolute path to the written provenance.json (already on disk). */
|
|
222
|
+
provenancePath: string;
|
|
223
|
+
/** Content fingerprint from the staged manifest. */
|
|
224
|
+
provenanceFingerprint: string;
|
|
225
|
+
/** Number of entries in the staged manifest. */
|
|
226
|
+
provenanceEntryCount: number;
|
|
227
|
+
/** The assembled model flag string (e.g. `--model claude-opus-4-8`). */
|
|
228
|
+
modelFlag: string;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Build the dry-run summary string. Pure function so it can be unit-tested
|
|
232
|
+
* without running the full harness.
|
|
233
|
+
*
|
|
234
|
+
* The summary covers three scenarios:
|
|
235
|
+
* 1. Declared (buildable) subject — no dist existed, fell back to source dir.
|
|
236
|
+
* 2. Declared (buildable) subject — existing dist was staged WITHOUT rebuilding
|
|
237
|
+
* (potentially stale).
|
|
238
|
+
* 3. Plain source subject — staged as-is, no build step.
|
|
239
|
+
*
|
|
240
|
+
* Always includes the assembled spawn command, the staged-manifest entry count +
|
|
241
|
+
* fingerprint, and the provenance.json path so a stale tree is visible at a glance.
|
|
242
|
+
*/
|
|
243
|
+
export declare function buildDryRunSummary(input: DryRunSummaryInput): string;
|
|
244
|
+
export interface CleanupHarnessOptions {
|
|
245
|
+
/** User asked to retain the harness dir (`--keep`) — never remove. */
|
|
246
|
+
keep: boolean;
|
|
247
|
+
/**
|
|
248
|
+
* True only when the harness itself created the dir under the OS tmp dir (no
|
|
249
|
+
* `--out`/`--workdir`). A user-supplied location is theirs to keep, so we only
|
|
250
|
+
* auto-remove the dir we created.
|
|
251
|
+
*/
|
|
252
|
+
created: boolean;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Remove the harness directory after a run so staged untrusted skill bytes and
|
|
256
|
+
* prompts do not accumulate in OS tmp. No-op when the user asked to keep it, when
|
|
257
|
+
* the dir is a user-supplied location (`--out`/`--workdir`), or when it is already
|
|
258
|
+
* gone. Idempotent and never throws — it runs from a `finally`, so it must not
|
|
259
|
+
* mask the run's real outcome.
|
|
260
|
+
*
|
|
261
|
+
* SAFETY: re-asserts the root is not a symlink immediately before removal (via
|
|
262
|
+
* `lstat`, which does NOT follow the link). A root swapped to a symlink between
|
|
263
|
+
* the run and cleanup is left in place rather than followed — `rmSync(recursive)`
|
|
264
|
+
* could otherwise delete the symlink's target outside tmp.
|
|
265
|
+
*/
|
|
266
|
+
export declare function cleanupHarness(harnessRoot: string, opts: CleanupHarnessOptions): void;
|
|
93
267
|
/**
|
|
94
268
|
* Domain orchestrator for `vat skill test run`. Pure of process.exit — all
|
|
95
269
|
* exit-code decisions live in the caller (run.ts).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-harness.d.ts","sourceRoot":"","sources":["../../src/skill-test/run-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;
|
|
1
|
+
{"version":3,"file":"run-harness.d.ts","sourceRoot":"","sources":["../../src/skill-test/run-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAe3E,OAAO,KAAK,EAAuB,yBAAyB,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAK5G,OAAO,EAIL,KAAK,sBAAsB,EAC5B,MAAM,iBAAiB,CAAC;AAWzB,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAsBhF,2CAA2C;AAC3C,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,cAAc,GAAG,SAAS,GAAG,MAAM,CAAC;AAE9E,mCAAmC;AACnC,MAAM,MAAM,oBAAoB,GAAG,cAAc,GAAG,SAAS,CAAC;AAE9D,MAAM,WAAW,iBAAiB;IAChC,mDAAmD;IACnD,MAAM,EAAE,MAAM,EAAE,CAAC;IAEjB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAEpD,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAErD,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,kEAAkE;IAClE,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,mEAAmE;IACnE,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,iEAAiE;IACjE,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,2CAA2C;IAC3C,IAAI,CAAC,EAAE,eAAe,CAAC;IAEvB,+BAA+B;IAC/B,WAAW,CAAC,EAAE,oBAAoB,CAAC;IAEnC,wCAAwC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,oDAAoD;IACpD,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB,2DAA2D;IAC3D,0BAA0B,CAAC,EAAE,OAAO,CAAC;IAErC,qDAAqD;IACrD,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,qEAAqE;IACrE,yBAAyB,CAAC,EAAE,OAAO,CAAC;IAGpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,aAAa,CAAC,EAAE,WAAW,CAAC;IAE5B;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,+EAA+E;IAC/E,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE5B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAUD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,uBAAuB;;;;CAI1B,CAAC;AAEX,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAEhE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,iBAAiB,EAAE,OAAO,GAAG,sBAAsB,CAEtG;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAE1E;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,iBAAiB,GAAG;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CASA;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,GACf,SAAS,CAAC,cAAc,CAAC,GAAG,SAAS,CAIvC;AAED,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,SAAS,GAAG,SAAS,GAC1B,SAAS,CAQX;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,CA2BtF;AAED,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,yBAAyB,CAMhG;AAkBD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEtD;AA4BD,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAAE,EACpB,IAAI,EAAE,iBAAiB,EACvB,KAAK,EAAE;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,GAC9B,cAAc,CA6BhB;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,MAAM,CAK3G;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,QAAQ,GAAG,2BAA2B,CAAC,GACpE,OAAO,CAET;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,WAAW,EAAE;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACpE,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,SAAS,EAAE,MAAM,GAChB,IAAI,CAUN;AAED;;;;;GAKG;AACH,uFAAuF;AACvF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAGhE;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAchH;AAkDD;;;;yBAIyB;AACzB,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAOpF;AAiFD,6CAA6C;AAC7C,MAAM,WAAW,kBAAkB;IACjC,uFAAuF;IACvF,UAAU,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,sEAAsE;IACtE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gDAAgD;IAChD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,CA+BpE;AAMD,MAAM,WAAW,qBAAqB;IACpC,sEAAsE;IACtE,IAAI,EAAE,OAAO,CAAC;IACd;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,GAAG,IAAI,CAcrF;AAMD;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAoR5F"}
|