@kontourai/flow-agents 3.3.0 → 3.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/add-to-project.yml +15 -0
- package/.github/workflows/ci.yml +161 -0
- package/CHANGELOG.md +48 -0
- package/CONTEXT.md +5 -1
- package/README.md +19 -8
- package/build/src/builder-flow-run-adapter.d.ts +80 -0
- package/build/src/builder-flow-run-adapter.js +241 -0
- package/build/src/builder-flow-runtime.d.ts +16 -0
- package/build/src/builder-flow-runtime.js +290 -0
- package/build/src/cli/builder-run.d.ts +1 -0
- package/build/src/cli/builder-run.js +27 -0
- package/build/src/cli/effective-backlog-settings.js +70 -2
- package/build/src/cli/init.d.ts +34 -0
- package/build/src/cli/init.js +341 -61
- package/build/src/cli/kit.js +55 -12
- package/build/src/cli/pull-work-provider.js +346 -5
- package/build/src/cli/skill-drift-check.d.ts +1 -0
- package/build/src/cli/skill-drift-check.js +165 -0
- package/build/src/cli/telemetry-doctor.d.ts +37 -0
- package/build/src/cli/telemetry-doctor.js +53 -6
- package/build/src/cli/validate-hook-influence.js +37 -7
- package/build/src/cli/workflow-sidecar.d.ts +93 -8
- package/build/src/cli/workflow-sidecar.js +1175 -158
- package/build/src/cli.js +5 -0
- package/build/src/flow-kit/validate.d.ts +54 -34
- package/build/src/flow-kit/validate.js +237 -26
- package/build/src/index.d.ts +2 -0
- package/build/src/index.js +1 -0
- package/build/src/lib/console-connect-options.d.ts +97 -0
- package/build/src/lib/console-connect-options.js +199 -0
- package/build/src/lib/console-telemetry-validate.d.ts +49 -0
- package/build/src/lib/console-telemetry-validate.js +91 -0
- package/build/src/lib/flow-resolver.d.ts +56 -3
- package/build/src/lib/flow-resolver.js +151 -11
- package/build/src/lib/fs.d.ts +17 -0
- package/build/src/lib/fs.js +172 -0
- package/build/src/lib/local-artifact-root.d.ts +44 -1
- package/build/src/lib/local-artifact-root.js +131 -3
- package/build/src/runtime-adapters.d.ts +39 -3
- package/build/src/runtime-adapters.js +77 -31
- package/build/src/tools/build-universal-bundles.js +40 -2
- package/build/src/tools/codex-agent-routing.d.ts +2 -0
- package/build/src/tools/codex-agent-routing.js +49 -0
- package/build/src/tools/generate-context-map.js +1 -0
- package/build/src/tools/validate-source-tree.js +27 -1
- package/context/scripts/hooks/lib/kit-catalog.js +235 -0
- package/context/scripts/hooks/lib/runnable-command.js +177 -0
- package/context/scripts/hooks/stop-goal-fit.js +278 -48
- package/context/scripts/hooks/workflow-steering.js +121 -21
- package/context/scripts/package.json +3 -0
- package/context/scripts/telemetry/install-console-config.sh +25 -4
- package/context/scripts/telemetry/lib/config.sh +102 -12
- package/context/scripts/telemetry/lib/pricing.sh +50 -0
- package/context/scripts/telemetry/lib/session.sh +3 -0
- package/context/scripts/telemetry/lib/transport.sh +87 -0
- package/context/scripts/telemetry/lib/usage.sh +205 -4
- package/context/scripts/telemetry/telemetry.conf +6 -0
- package/context/scripts/telemetry/telemetry.sh +48 -0
- package/context/settings/workspace-backlog-provider-settings.example.json +48 -0
- package/docs/agent-usage-feedback-loop.md +35 -0
- package/docs/architecture-engine-and-kits.md +110 -0
- package/docs/context-map.md +2 -0
- package/docs/decisions/embeddable-engine.md +152 -0
- package/docs/decisions/index.md +3 -1
- package/docs/decisions/trust-ledger-retention.md +88 -0
- package/docs/decisions/workflow-enforcement.md +31 -9
- package/docs/fixture-ownership.md +3 -0
- package/docs/implementing-trust-reconciliation.md +129 -0
- package/docs/index.md +19 -9
- package/docs/integrations/flow-agents-console.md +167 -0
- package/docs/kit-authoring-guide.md +52 -21
- package/docs/spec/builder-flow-runtime.md +80 -0
- package/docs/spec/runtime-hook-surface.md +45 -1
- package/docs/specs/economics-record-contract.md +270 -0
- package/docs/specs/harness-capability-matrix.md +74 -0
- package/docs/specs/learning-review-proposals-contract.md +340 -0
- package/docs/specs/routing-efficiency-review.md +59 -0
- package/docs/verifiable-trust.md +74 -25
- package/docs/workflow-usage-guide.md +10 -0
- package/evals/acceptance/prove-capture-teeth.sh +132 -0
- package/evals/ci/antigaming-suite.sh +1 -0
- package/evals/ci/run-baseline.sh +72 -4
- package/evals/fixtures/economics/acceptance.json +12 -0
- package/evals/fixtures/economics/agents/tool-worker-1/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-2/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-3/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-4/events.jsonl +1 -0
- package/evals/fixtures/economics/agents/tool-worker-5/events.jsonl +2 -0
- package/evals/fixtures/economics/critique.json +22 -0
- package/evals/fixtures/economics/expected-record.json +71 -0
- package/evals/fixtures/economics/session-usage-event.json +1 -0
- package/evals/fixtures/economics/state.json +11 -0
- package/evals/fixtures/economics/transcript.jsonl +3 -0
- package/evals/fixtures/hook-influence/cases.json +7 -7
- package/evals/fixtures/learning-review-proposals/balanced/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/economics.jsonl +5 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.json +30 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.md +66 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/repeat-window/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/under-threshold/economics.jsonl +3 -0
- package/evals/fixtures/telemetry/usage-transcript-sample.jsonl +4 -0
- package/evals/fixtures/trust-reconcile-exploits/mcp-degrade.json +42 -0
- package/evals/integration/test_builder_entry_enforcement.sh +241 -0
- package/evals/integration/test_builder_step_producers.sh +18 -10
- package/evals/integration/test_bundle_install.sh +172 -0
- package/evals/integration/test_console_tenant_isolation.sh +167 -0
- package/evals/integration/test_critique_supersession_roundtrip.sh +4 -1
- package/evals/integration/test_dual_emit_flow_step.sh +10 -4
- package/evals/integration/test_economics_record.sh +674 -0
- package/evals/integration/test_effective_backlog_settings.sh +1 -1
- package/evals/integration/test_evidence_capture_hook.sh +17 -2
- package/evals/integration/test_exemption_usage_review.sh +198 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +83 -0
- package/evals/integration/test_flowdef_session_activation.sh +0 -1
- package/evals/integration/test_flowdef_session_history_preservation.sh +13 -3
- package/evals/integration/test_gate_lockdown.sh +7 -0
- package/evals/integration/test_gate_review_inquiry_records.sh +9 -1
- package/evals/integration/test_goal_fit_hook.sh +2031 -0
- package/evals/integration/test_hook_category_behaviors.sh +8 -1
- package/evals/integration/test_hook_influence_cases.sh +25 -1
- package/evals/integration/test_install_merge.sh +227 -2
- package/evals/integration/test_kit_conformance_levels.sh +6 -6
- package/evals/integration/test_learning_review_proposals.sh +329 -0
- package/evals/integration/test_liveness_conflict_injection.sh +26 -22
- package/evals/integration/test_liveness_console_relay.sh +166 -0
- package/evals/integration/test_liveness_heartbeat.sh +17 -17
- package/evals/integration/test_liveness_worktree_root.sh +575 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +6 -1
- package/evals/integration/test_publish_delivery.sh +331 -1
- package/evals/integration/test_pull_work_board.sh +200 -0
- package/evals/integration/test_pull_work_provider.sh +1 -1
- package/evals/integration/test_record_check.sh +378 -0
- package/evals/integration/test_routing_efficiency.sh +71 -0
- package/evals/integration/test_runtime_adapter_activation.sh +28 -0
- package/evals/integration/test_session_resume_roundtrip.sh +16 -19
- package/evals/integration/test_skill_drift_check.sh +870 -0
- package/evals/integration/test_telemetry.sh +445 -0
- package/evals/integration/test_telemetry_doctor.sh +66 -0
- package/evals/integration/test_telemetry_usage_pipeline.sh +228 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +30 -13
- package/evals/integration/test_trust_reconcile_trailer_diagnostic.sh +247 -0
- package/evals/integration/test_usage_cost.sh +61 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +1395 -0
- package/evals/integration/test_workflow_steering_hook.sh +157 -16
- package/evals/integration/test_workspace_settings.sh +176 -0
- package/evals/lib/env.sh +26 -0
- package/evals/lib/node.sh +8 -0
- package/evals/run.sh +29 -0
- package/evals/static/test_ci_integration_coverage.sh +115 -0
- package/evals/static/test_declared_scope_forms_documented.sh +114 -0
- package/evals/static/test_universal_bundles.sh +34 -0
- package/evals/static/test_validate_source_kit_asset_scope.sh +259 -0
- package/evals/static/test_workflow_skills.sh +1 -1
- package/kits/builder/flows/build.flow.json +9 -18
- package/kits/builder/flows/publish-learn.flow.json +5 -1
- package/kits/builder/kit.json +120 -0
- package/kits/builder/skills/deliver/SKILL.md +42 -0
- package/kits/builder/skills/evidence-gate/SKILL.md +12 -0
- package/kits/builder/skills/execute-plan/SKILL.md +9 -0
- package/kits/builder/skills/learning-review/SKILL.md +51 -0
- package/kits/builder/skills/plan-work/SKILL.md +17 -20
- package/kits/builder/skills/pull-work/SKILL.md +21 -0
- package/kits/builder/skills/release-readiness/SKILL.md +12 -0
- package/kits/knowledge/kit.json +9 -0
- package/kits/veritas-governance/docs/README.md +35 -7
- package/kits/veritas-governance/fixtures/exemption-review/mixed-fresh-stale.DECLARED.json +14 -0
- package/kits/veritas-governance/kit.json +14 -0
- package/kits/veritas-governance/skills/exemption-usage-review/SKILL.md +128 -0
- package/kits/veritas-governance/skills/exemption-usage-review/review-exemptions.mjs +231 -0
- package/package.json +2 -2
- package/packaging/manifest.json +29 -0
- package/schemas/backlog-provider-settings.schema.json +13 -0
- package/schemas/workflow-state.schema.json +44 -0
- package/scripts/README.md +4 -0
- package/scripts/check-content-boundary.cjs +8 -1
- package/scripts/ci/trust-reconcile.js +136 -0
- package/scripts/hooks/codex-hook-adapter.js +77 -2
- package/scripts/hooks/evidence-capture.js +38 -5
- package/scripts/hooks/lib/codex-exit-code.js +316 -0
- package/scripts/hooks/lib/kit-catalog.js +235 -0
- package/scripts/hooks/lib/liveness-write.js +28 -1
- package/scripts/hooks/lib/local-artifact-paths.js +97 -1
- package/scripts/hooks/lib/runnable-command.js +177 -0
- package/scripts/hooks/lib/skill-drift.js +350 -0
- package/scripts/hooks/stop-goal-fit.js +278 -48
- package/scripts/hooks/workflow-steering.js +121 -21
- package/scripts/install-codex-home.sh +97 -47
- package/scripts/install-merge.js +72 -14
- package/scripts/install-owned-files.js +178 -0
- package/scripts/liveness/relay.sh +84 -0
- package/scripts/telemetry/economics-record.schema.json +145 -0
- package/scripts/telemetry/economics-record.sh +331 -0
- package/scripts/telemetry/install-console-config.sh +25 -4
- package/scripts/telemetry/learning-review-decide.sh +124 -0
- package/scripts/telemetry/learning-review-proposals.schema.json +161 -0
- package/scripts/telemetry/learning-review-proposals.sh +484 -0
- package/scripts/telemetry/lib/config.sh +102 -12
- package/scripts/telemetry/lib/pricing.sh +14 -6
- package/scripts/telemetry/lib/session.sh +3 -0
- package/scripts/telemetry/lib/transport.sh +133 -15
- package/scripts/telemetry/lib/usage.sh +121 -28
- package/scripts/telemetry/routing-efficiency.sh +0 -0
- package/scripts/telemetry/telemetry.conf +6 -0
- package/scripts/telemetry/telemetry.sh +48 -0
- package/src/builder-flow-run-adapter.ts +357 -0
- package/src/builder-flow-runtime.ts +348 -0
- package/src/cli/builder-flow-run-adapter.test.mjs +495 -0
- package/src/cli/builder-flow-runtime.test.mjs +213 -0
- package/src/cli/builder-run.ts +28 -0
- package/src/cli/codex-agent-routing.test.mjs +44 -0
- package/src/cli/codex-exit-code.test.mjs +207 -0
- package/src/cli/console-connect-options.test.mjs +329 -0
- package/src/cli/console-telemetry-validate.test.mjs +157 -0
- package/src/cli/effective-backlog-settings.ts +68 -2
- package/src/cli/flow-resolver-composition.test.mjs +101 -0
- package/src/cli/init.test.mjs +161 -0
- package/src/cli/init.ts +407 -62
- package/src/cli/kit-metadata-security.test.mjs +443 -0
- package/src/cli/kit.ts +50 -12
- package/src/cli/pull-work-provider.ts +377 -3
- package/src/cli/sidecar-pure-helpers.test.mjs +64 -0
- package/src/cli/skill-drift-check.ts +196 -0
- package/src/cli/telemetry-doctor.test.mjs +53 -0
- package/src/cli/telemetry-doctor.ts +50 -7
- package/src/cli/validate-hook-influence.ts +37 -6
- package/src/cli/workflow-sidecar.ts +1150 -151
- package/src/cli.ts +5 -0
- package/src/flow-kit/validate.ts +277 -38
- package/src/index.ts +19 -0
- package/src/lib/console-connect-options.ts +261 -0
- package/src/lib/console-telemetry-validate.ts +88 -0
- package/src/lib/flow-resolver.ts +153 -10
- package/src/lib/fs.ts +160 -0
- package/src/lib/local-artifact-root.ts +129 -3
- package/src/runtime-adapters.ts +113 -33
- package/src/tools/build-universal-bundles.ts +36 -2
- package/src/tools/codex-agent-routing.ts +48 -0
- package/src/tools/generate-context-map.ts +1 -0
- package/src/tools/validate-source-tree.ts +26 -1
|
@@ -0,0 +1,870 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_skill_drift_check.sh — Fixture-based integration eval for installed-skill drift
|
|
3
|
+
# detection (kontourai/flow-agents#439, slice 1).
|
|
4
|
+
#
|
|
5
|
+
# Covers:
|
|
6
|
+
# 1. Baseline scenario: no manifest / no installed skills dir yet -> exit 2 (cannot check),
|
|
7
|
+
# output explains why.
|
|
8
|
+
# 2. Simulated install (manifest + `cp -R`, deliberately decoupled from the full
|
|
9
|
+
# `init --global` bundle-build path — that path is proven by test_install_merge.sh's
|
|
10
|
+
# Scenario 5) -> exit 0, every file in_sync.
|
|
11
|
+
# 3. Kit-updated scenario: kit source changes, installed copy + manifest baseline unchanged
|
|
12
|
+
# -> exit 1, the changed file reported kit_updated, the rest in_sync, output names the
|
|
13
|
+
# literal refresh command.
|
|
14
|
+
# 4. User-modified scenario: an installed file is edited locally (kit source reverted to the
|
|
15
|
+
# unchanged baseline) -> exit 1, that file reported user_modified (never kit_updated), and
|
|
16
|
+
# its bytes/mtime are asserted byte-identical before and after the check ran (read-only
|
|
17
|
+
# proof, not merely "no writeFileSync call visible").
|
|
18
|
+
# 5. Read-only proof: every invocation above captures a recursive checksum of the installed
|
|
19
|
+
# skills dir and the kit-source dir immediately before and after the call, asserting
|
|
20
|
+
# byte-identical — the check must never write under either directory.
|
|
21
|
+
# 6. SessionStart advisory scenario: `scripts/hooks/workflow-steering.js`'s exported `run()`
|
|
22
|
+
# surfaces a `[SKILL DRIFT]` line when invoked against a kit-bearing fixture repo with
|
|
23
|
+
# drift, and omits it entirely when the fixture is clean.
|
|
24
|
+
# 7. Unbaselined-only scenario (review fix, blind spot that let the FIX1 exit-code regression
|
|
25
|
+
# escape review): manifest present but missing exactly one file's entry, that file's
|
|
26
|
+
# installed bytes differ from kit source -> exit 1, state "unbaselined".
|
|
27
|
+
# 8. Missing-install-only scenario (review fix): a kit-source file with no installed
|
|
28
|
+
# counterpart at all (manifest entry for it is irrelevant/absent) -> exit 1, state
|
|
29
|
+
# "missing_install".
|
|
30
|
+
# 9. Kit-removed scenario (review fix, new fifth/sixth drift state): an installed file that
|
|
31
|
+
# still matches its last recorded manifest baseline exactly, but no longer exists anywhere
|
|
32
|
+
# in the current kit source -> exit 1, state "kit_removed", output carries the
|
|
33
|
+
# "refresh will NOT delete it" guidance (never the generic kit_updated wording).
|
|
34
|
+
# 10. Corrupt-manifest SessionStart scenario (review fix): an unparseable/invalid-JSON manifest
|
|
35
|
+
# file paired with an otherwise-clean install (installed bytes already match kit source
|
|
36
|
+
# byte-for-byte) -> the SessionStart advisory must not crash and must not emit a
|
|
37
|
+
# `[SKILL DRIFT]` line (loadManifest's parse-failure tolerance + in_sync-regardless-of-
|
|
38
|
+
# manifest classification both hold).
|
|
39
|
+
# 11. Foreign-installed-file scenario (kontourai/flow-agents#465 fix): a file installed in the
|
|
40
|
+
# same shared dest but never owned by any kit source and never recorded in any manifest
|
|
41
|
+
# (e.g. another tool's skill dir) is outside `compareSkillDrift`'s jurisdiction entirely --
|
|
42
|
+
# (a) kit files in sync -> exit 0, foreign path absent from both plain and --json output;
|
|
43
|
+
# (b) one drifted kit file -> exit 1 listing only the kit file, foreign path still absent;
|
|
44
|
+
# (c) SessionStart advisory with the foreign file present and kit in sync -> no
|
|
45
|
+
# `[SKILL DRIFT]` line. The seeded foreign file stays inside the read-only checksum
|
|
46
|
+
# bracketing throughout (a)/(b).
|
|
47
|
+
#
|
|
48
|
+
# Isolation idioms mirrored verbatim from test_install_merge.sh (this directory's sibling eval): mktemp -d,
|
|
49
|
+
# trap cleanup EXIT, _pass/_fail counters, FLOW_AGENTS_USER_CLAUDE_SETTINGS for destination
|
|
50
|
+
# isolation (used in Scenarios 6 and 10, where the SessionStart advisory resolves its dest purely
|
|
51
|
+
# from that env var — the CLI check itself is exercised via its own `--dest` flag elsewhere).
|
|
52
|
+
#
|
|
53
|
+
# No network calls. All kit-source fixtures are synthetic directories created under
|
|
54
|
+
# $TMPDIR_EVAL, reached via `skill-drift-check`'s `--kit-source-dir` override — this eval never
|
|
55
|
+
# reads or writes this repo's own `kits/` or `dist/` trees.
|
|
56
|
+
set -uo pipefail
|
|
57
|
+
|
|
58
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
59
|
+
source "$ROOT_DIR/evals/lib/env.sh"
|
|
60
|
+
flow_agents_eval_bootstrap "$ROOT_DIR" || exit $?
|
|
61
|
+
CLI="$ROOT_DIR/build/src/cli.js"
|
|
62
|
+
SKILL_DRIFT_LIB="$ROOT_DIR/scripts/hooks/lib/skill-drift.js"
|
|
63
|
+
WORKFLOW_STEERING="$ROOT_DIR/scripts/hooks/workflow-steering.js"
|
|
64
|
+
TMPDIR_EVAL="$(mktemp -d /tmp/skill-drift-check.XXXXXX)"
|
|
65
|
+
pass=0
|
|
66
|
+
fail=0
|
|
67
|
+
|
|
68
|
+
cleanup() {
|
|
69
|
+
rm -rf "$TMPDIR_EVAL"
|
|
70
|
+
}
|
|
71
|
+
trap cleanup EXIT
|
|
72
|
+
|
|
73
|
+
_pass() { echo " ✓ $1"; pass=$((pass + 1)); }
|
|
74
|
+
_fail() { echo " ✗ $1"; fail=$((fail + 1)); }
|
|
75
|
+
|
|
76
|
+
# Recursive content checksum of a directory (sorted, so unrelated ordering differences don't
|
|
77
|
+
# cause false failures). Reports a sentinel string if the directory does not exist, so a
|
|
78
|
+
# before/after comparison across a directory's creation is still a meaningful (unequal) result
|
|
79
|
+
# rather than a shell error.
|
|
80
|
+
_dirsum() {
|
|
81
|
+
local dir="$1"
|
|
82
|
+
if [[ -d "$dir" ]]; then
|
|
83
|
+
find "$dir" -type f -exec cksum {} \; | sort
|
|
84
|
+
else
|
|
85
|
+
echo "MISSING:$dir"
|
|
86
|
+
fi
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
echo "=== Skill Drift Check Integration Tests (#439) ==="
|
|
90
|
+
echo ""
|
|
91
|
+
|
|
92
|
+
echo "--- Build ---"
|
|
93
|
+
# Always rebuild (never only-when-missing): a stale build/ from a previous run must never mask
|
|
94
|
+
# TypeScript changes to skill-drift-check.ts/init.ts under test here (#439 review fix — this eval
|
|
95
|
+
# blind spot is exactly what let the FIX1 exit-code regression escape review the first time).
|
|
96
|
+
if (cd "$ROOT_DIR" && npm run build --silent >/dev/null 2>&1); then
|
|
97
|
+
_pass "TypeScript build completed"
|
|
98
|
+
else
|
|
99
|
+
_fail "TypeScript build failed"
|
|
100
|
+
echo "Results: 0/$((pass + fail + 1)) passed, $((fail + 1)) failed"
|
|
101
|
+
exit 1
|
|
102
|
+
fi
|
|
103
|
+
echo ""
|
|
104
|
+
|
|
105
|
+
# ─── Fixture: synthetic 3-skill kit source A ─────────────────────────────────
|
|
106
|
+
KIT_A="$TMPDIR_EVAL/kit-source-a"
|
|
107
|
+
mkdir -p "$KIT_A/skill-one" "$KIT_A/skill-two" "$KIT_A/skill-three"
|
|
108
|
+
printf '# Skill One\ncontent-one\n' > "$KIT_A/skill-one/SKILL.md"
|
|
109
|
+
printf '# Skill Two\ncontent-two\n' > "$KIT_A/skill-two/SKILL.md"
|
|
110
|
+
printf '# Skill Three\ncontent-three\n' > "$KIT_A/skill-three/SKILL.md"
|
|
111
|
+
|
|
112
|
+
DEST="$TMPDIR_EVAL/dest"
|
|
113
|
+
|
|
114
|
+
# ─── Scenario 1: Baseline — no manifest, no installed skills dir ─────────────
|
|
115
|
+
echo "--- Scenario 1: Baseline (no manifest, no installed skills dir) -> exit 2 ---"
|
|
116
|
+
|
|
117
|
+
BEFORE_KIT_A=$(_dirsum "$KIT_A")
|
|
118
|
+
S1_OUT="$TMPDIR_EVAL/s1.out"
|
|
119
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST" > "$S1_OUT" 2>&1; then
|
|
120
|
+
S1_RC=0
|
|
121
|
+
else
|
|
122
|
+
S1_RC=$?
|
|
123
|
+
fi
|
|
124
|
+
AFTER_KIT_A=$(_dirsum "$KIT_A")
|
|
125
|
+
|
|
126
|
+
if [[ "$S1_RC" -eq 2 ]]; then
|
|
127
|
+
_pass "baseline: exit code 2 (cannot fully check)"
|
|
128
|
+
else
|
|
129
|
+
_fail "baseline: expected exit 2, got $S1_RC"; cat "$S1_OUT"
|
|
130
|
+
fi
|
|
131
|
+
|
|
132
|
+
if grep -q "cannot fully check" "$S1_OUT" && grep -q "no manifest found" "$S1_OUT" && grep -q "installed skills directory does not exist" "$S1_OUT"; then
|
|
133
|
+
_pass "baseline: output explains missing manifest and missing installed dir"
|
|
134
|
+
else
|
|
135
|
+
_fail "baseline: output did not explain the cannot-check reason"; cat "$S1_OUT"
|
|
136
|
+
fi
|
|
137
|
+
|
|
138
|
+
if [[ "$BEFORE_KIT_A" == "$AFTER_KIT_A" ]]; then
|
|
139
|
+
_pass "baseline: kit source A unchanged (read-only)"
|
|
140
|
+
else
|
|
141
|
+
_fail "baseline: kit source A was modified by the check"
|
|
142
|
+
fi
|
|
143
|
+
if [[ ! -e "$DEST" ]]; then
|
|
144
|
+
_pass "baseline: check did not create the dest directory"
|
|
145
|
+
else
|
|
146
|
+
_fail "baseline: check created $DEST despite exiting cannot-check"
|
|
147
|
+
fi
|
|
148
|
+
|
|
149
|
+
echo ""
|
|
150
|
+
|
|
151
|
+
# ─── Scenario 2: Simulated install — manifest + cp, decoupled from init --global ─
|
|
152
|
+
echo "--- Scenario 2: Simulated install -> exit 0, all in_sync ---"
|
|
153
|
+
|
|
154
|
+
node -e "
|
|
155
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
156
|
+
const manifest = buildManifest({ skillsSourceDir: '$KIT_A', runtime: 'claude-code' });
|
|
157
|
+
writeManifestAtomic('$DEST/.flow-agents/skills-manifest.json', manifest);
|
|
158
|
+
"
|
|
159
|
+
mkdir -p "$DEST/skills"
|
|
160
|
+
cp -R "$KIT_A/." "$DEST/skills/"
|
|
161
|
+
|
|
162
|
+
if [[ -f "$DEST/.flow-agents/skills-manifest.json" ]]; then
|
|
163
|
+
_pass "simulated install: manifest written at \$DEST/.flow-agents/skills-manifest.json"
|
|
164
|
+
else
|
|
165
|
+
_fail "simulated install: manifest missing"
|
|
166
|
+
fi
|
|
167
|
+
|
|
168
|
+
BEFORE_DEST=$(_dirsum "$DEST/skills")
|
|
169
|
+
BEFORE_KIT_A=$(_dirsum "$KIT_A")
|
|
170
|
+
S2_OUT="$TMPDIR_EVAL/s2.out"
|
|
171
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST" > "$S2_OUT" 2>&1; then
|
|
172
|
+
S2_RC=0
|
|
173
|
+
else
|
|
174
|
+
S2_RC=$?
|
|
175
|
+
fi
|
|
176
|
+
S2_JSON="$TMPDIR_EVAL/s2.json"
|
|
177
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST" --json > "$S2_JSON" 2>&1; then
|
|
178
|
+
S2_JSON_RC=0
|
|
179
|
+
else
|
|
180
|
+
S2_JSON_RC=$?
|
|
181
|
+
fi
|
|
182
|
+
AFTER_DEST=$(_dirsum "$DEST/skills")
|
|
183
|
+
AFTER_KIT_A=$(_dirsum "$KIT_A")
|
|
184
|
+
|
|
185
|
+
if [[ "$S2_RC" -eq 0 && "$S2_JSON_RC" -eq 0 ]]; then
|
|
186
|
+
_pass "clean install: both plain and --json runs exit 0"
|
|
187
|
+
else
|
|
188
|
+
_fail "clean install: expected exit 0/0, got plain=$S2_RC json=$S2_JSON_RC"; cat "$S2_OUT"; cat "$S2_JSON"
|
|
189
|
+
fi
|
|
190
|
+
|
|
191
|
+
if grep -q "All installed skill files are in sync" "$S2_OUT"; then
|
|
192
|
+
_pass "clean install: plain output reports all files in sync"
|
|
193
|
+
else
|
|
194
|
+
_fail "clean install: plain output did not report in-sync summary"; cat "$S2_OUT"
|
|
195
|
+
fi
|
|
196
|
+
|
|
197
|
+
if node - "$S2_JSON" << 'NODE'
|
|
198
|
+
const fs = require("node:fs");
|
|
199
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
200
|
+
if (report.hasDrift !== false) throw new Error("hasDrift should be false: " + JSON.stringify(report.summary));
|
|
201
|
+
if (report.summary.total !== 3) throw new Error("expected 3 files total, got " + report.summary.total);
|
|
202
|
+
if (report.summary.inSync !== 3) throw new Error("expected 3 in_sync, got " + report.summary.inSync);
|
|
203
|
+
if (!report.files.every((f) => f.state === "in_sync")) throw new Error("not every file is in_sync: " + JSON.stringify(report.files));
|
|
204
|
+
console.log("ok");
|
|
205
|
+
NODE
|
|
206
|
+
then
|
|
207
|
+
_pass "clean install: --json report classifies all 3 files in_sync, hasDrift=false"
|
|
208
|
+
else
|
|
209
|
+
_fail "clean install: --json report did not classify all files in_sync"
|
|
210
|
+
fi
|
|
211
|
+
|
|
212
|
+
if [[ "$BEFORE_DEST" == "$AFTER_DEST" ]]; then
|
|
213
|
+
_pass "clean install: installed skills dir unchanged across both check invocations (read-only)"
|
|
214
|
+
else
|
|
215
|
+
_fail "clean install: installed skills dir was modified by the check"
|
|
216
|
+
fi
|
|
217
|
+
if [[ "$BEFORE_KIT_A" == "$AFTER_KIT_A" ]]; then
|
|
218
|
+
_pass "clean install: kit source A unchanged across both check invocations (read-only)"
|
|
219
|
+
else
|
|
220
|
+
_fail "clean install: kit source A was modified by the check"
|
|
221
|
+
fi
|
|
222
|
+
|
|
223
|
+
echo ""
|
|
224
|
+
|
|
225
|
+
# ─── Scenario 3: Kit-updated — kit source changes, installed/manifest unchanged ─
|
|
226
|
+
echo "--- Scenario 3: Kit-updated (kit source changed, installed+manifest unchanged) -> exit 1 ---"
|
|
227
|
+
|
|
228
|
+
KIT_B="$TMPDIR_EVAL/kit-source-b"
|
|
229
|
+
cp -R "$KIT_A" "$KIT_B"
|
|
230
|
+
printf '# Skill One\ncontent-one-KIT-UPDATED\n' > "$KIT_B/skill-one/SKILL.md"
|
|
231
|
+
|
|
232
|
+
BEFORE_DEST=$(_dirsum "$DEST/skills")
|
|
233
|
+
BEFORE_KIT_B=$(_dirsum "$KIT_B")
|
|
234
|
+
S3_OUT="$TMPDIR_EVAL/s3.out"
|
|
235
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_B" --dest "$DEST" > "$S3_OUT" 2>&1; then
|
|
236
|
+
S3_RC=0
|
|
237
|
+
else
|
|
238
|
+
S3_RC=$?
|
|
239
|
+
fi
|
|
240
|
+
S3_JSON="$TMPDIR_EVAL/s3.json"
|
|
241
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_B" --dest "$DEST" --json > "$S3_JSON" 2>&1; then
|
|
242
|
+
S3_JSON_RC=0
|
|
243
|
+
else
|
|
244
|
+
S3_JSON_RC=$?
|
|
245
|
+
fi
|
|
246
|
+
AFTER_DEST=$(_dirsum "$DEST/skills")
|
|
247
|
+
AFTER_KIT_B=$(_dirsum "$KIT_B")
|
|
248
|
+
|
|
249
|
+
if [[ "$S3_RC" -eq 1 && "$S3_JSON_RC" -eq 1 ]]; then
|
|
250
|
+
_pass "kit-updated: both plain and --json runs exit 1"
|
|
251
|
+
else
|
|
252
|
+
_fail "kit-updated: expected exit 1/1, got plain=$S3_RC json=$S3_JSON_RC"; cat "$S3_OUT"; cat "$S3_JSON"
|
|
253
|
+
fi
|
|
254
|
+
|
|
255
|
+
if grep -q "kit_updated" "$S3_OUT" && grep -q "skill-one/SKILL.md" "$S3_OUT"; then
|
|
256
|
+
_pass "kit-updated: plain output lists skill-one/SKILL.md under kit_updated"
|
|
257
|
+
else
|
|
258
|
+
_fail "kit-updated: plain output did not list the changed file under kit_updated"; cat "$S3_OUT"
|
|
259
|
+
fi
|
|
260
|
+
|
|
261
|
+
if grep -qF "flow-agents init --runtime claude-code --global" "$S3_OUT"; then
|
|
262
|
+
_pass "kit-updated: plain output names the literal refresh command"
|
|
263
|
+
else
|
|
264
|
+
_fail "kit-updated: plain output did not name the refresh command"; cat "$S3_OUT"
|
|
265
|
+
fi
|
|
266
|
+
|
|
267
|
+
if node - "$S3_JSON" << 'NODE'
|
|
268
|
+
const fs = require("node:fs");
|
|
269
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
270
|
+
if (report.hasDrift !== true) throw new Error("hasDrift should be true: " + JSON.stringify(report.summary));
|
|
271
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
272
|
+
if (byPath["skill-one/SKILL.md"] !== "kit_updated") throw new Error("skill-one/SKILL.md should be kit_updated, got " + byPath["skill-one/SKILL.md"]);
|
|
273
|
+
if (byPath["skill-two/SKILL.md"] !== "in_sync") throw new Error("skill-two/SKILL.md should remain in_sync, got " + byPath["skill-two/SKILL.md"]);
|
|
274
|
+
if (byPath["skill-three/SKILL.md"] !== "in_sync") throw new Error("skill-three/SKILL.md should remain in_sync, got " + byPath["skill-three/SKILL.md"]);
|
|
275
|
+
console.log("ok");
|
|
276
|
+
NODE
|
|
277
|
+
then
|
|
278
|
+
_pass "kit-updated: --json report classifies exactly skill-one/SKILL.md as kit_updated, others in_sync"
|
|
279
|
+
else
|
|
280
|
+
_fail "kit-updated: --json report classification incorrect"
|
|
281
|
+
fi
|
|
282
|
+
|
|
283
|
+
if [[ "$BEFORE_DEST" == "$AFTER_DEST" ]]; then
|
|
284
|
+
_pass "kit-updated: installed skills dir unchanged across both check invocations (read-only)"
|
|
285
|
+
else
|
|
286
|
+
_fail "kit-updated: installed skills dir was modified by the check"
|
|
287
|
+
fi
|
|
288
|
+
if [[ "$BEFORE_KIT_B" == "$AFTER_KIT_B" ]]; then
|
|
289
|
+
_pass "kit-updated: kit source B unchanged across both check invocations (read-only)"
|
|
290
|
+
else
|
|
291
|
+
_fail "kit-updated: kit source B was modified by the check"
|
|
292
|
+
fi
|
|
293
|
+
|
|
294
|
+
echo ""
|
|
295
|
+
|
|
296
|
+
# ─── Scenario 4: User-modified — installed file edited locally ──────────────
|
|
297
|
+
echo "--- Scenario 4: User-modified (installed file edited locally) -> exit 1 ---"
|
|
298
|
+
|
|
299
|
+
printf '# Skill Two\ncontent-two-USER-EDITED\n' > "$DEST/sk""ills/skill-two/SKILL.md"
|
|
300
|
+
USER_FILE_MTIME_BEFORE=$(stat -c %Y "$DEST/sk""ills/skill-two/SKILL.md" 2>/dev/null || stat -f %m "$DEST/sk""ills/skill-two/SKILL.md")
|
|
301
|
+
USER_FILE_SUM_BEFORE=$(cksum "$DEST/sk""ills/skill-two/SKILL.md")
|
|
302
|
+
|
|
303
|
+
BEFORE_DEST=$(_dirsum "$DEST/skills")
|
|
304
|
+
BEFORE_KIT_A=$(_dirsum "$KIT_A")
|
|
305
|
+
S4_OUT="$TMPDIR_EVAL/s4.out"
|
|
306
|
+
# Kit source reverted to the unchanged baseline A (installed dir + manifest also unchanged
|
|
307
|
+
# from Scenario 2's simulated install) — isolates the user-edit as the only variable.
|
|
308
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST" > "$S4_OUT" 2>&1; then
|
|
309
|
+
S4_RC=0
|
|
310
|
+
else
|
|
311
|
+
S4_RC=$?
|
|
312
|
+
fi
|
|
313
|
+
S4_JSON="$TMPDIR_EVAL/s4.json"
|
|
314
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST" --json > "$S4_JSON" 2>&1; then
|
|
315
|
+
S4_JSON_RC=0
|
|
316
|
+
else
|
|
317
|
+
S4_JSON_RC=$?
|
|
318
|
+
fi
|
|
319
|
+
AFTER_DEST=$(_dirsum "$DEST/skills")
|
|
320
|
+
AFTER_KIT_A=$(_dirsum "$KIT_A")
|
|
321
|
+
USER_FILE_MTIME_AFTER=$(stat -c %Y "$DEST/sk""ills/skill-two/SKILL.md" 2>/dev/null || stat -f %m "$DEST/sk""ills/skill-two/SKILL.md")
|
|
322
|
+
USER_FILE_SUM_AFTER=$(cksum "$DEST/sk""ills/skill-two/SKILL.md")
|
|
323
|
+
|
|
324
|
+
if [[ "$S4_RC" -eq 1 && "$S4_JSON_RC" -eq 1 ]]; then
|
|
325
|
+
_pass "user-modified: both plain and --json runs exit 1"
|
|
326
|
+
else
|
|
327
|
+
_fail "user-modified: expected exit 1/1, got plain=$S4_RC json=$S4_JSON_RC"; cat "$S4_OUT"; cat "$S4_JSON"
|
|
328
|
+
fi
|
|
329
|
+
|
|
330
|
+
if grep -q "user_modified" "$S4_OUT" && grep -q "skill-two/SKILL.md" "$S4_OUT"; then
|
|
331
|
+
_pass "user-modified: plain output lists skill-two/SKILL.md under user_modified"
|
|
332
|
+
else
|
|
333
|
+
_fail "user-modified: plain output did not list the edited file under user_modified"; cat "$S4_OUT"
|
|
334
|
+
fi
|
|
335
|
+
|
|
336
|
+
if node - "$S4_JSON" << 'NODE'
|
|
337
|
+
const fs = require("node:fs");
|
|
338
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
339
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
340
|
+
if (byPath["skill-two/SKILL.md"] !== "user_modified") throw new Error("skill-two/SKILL.md should be user_modified, got " + byPath["skill-two/SKILL.md"]);
|
|
341
|
+
if (byPath["skill-one/SKILL.md"] !== "in_sync") throw new Error("skill-one/SKILL.md should be in_sync (kit source reverted to A), got " + byPath["skill-one/SKILL.md"]);
|
|
342
|
+
if (byPath["skill-three/SKILL.md"] !== "in_sync") throw new Error("skill-three/SKILL.md should remain in_sync, got " + byPath["skill-three/SKILL.md"]);
|
|
343
|
+
console.log("ok");
|
|
344
|
+
NODE
|
|
345
|
+
then
|
|
346
|
+
_pass "user-modified: --json report classifies exactly skill-two/SKILL.md as user_modified (never kit_updated), others in_sync"
|
|
347
|
+
else
|
|
348
|
+
_fail "user-modified: --json report classification incorrect"
|
|
349
|
+
fi
|
|
350
|
+
|
|
351
|
+
if [[ "$USER_FILE_SUM_BEFORE" == "$USER_FILE_SUM_AFTER" ]]; then
|
|
352
|
+
_pass "user-modified: edited file bytes unchanged after the check ran (not silently overwritten)"
|
|
353
|
+
else
|
|
354
|
+
_fail "user-modified: edited file bytes changed after the check ran"
|
|
355
|
+
fi
|
|
356
|
+
if [[ "$USER_FILE_MTIME_BEFORE" == "$USER_FILE_MTIME_AFTER" ]]; then
|
|
357
|
+
_pass "user-modified: edited file mtime unchanged after the check ran"
|
|
358
|
+
else
|
|
359
|
+
_fail "user-modified: edited file mtime changed after the check ran ($USER_FILE_MTIME_BEFORE -> $USER_FILE_MTIME_AFTER)"
|
|
360
|
+
fi
|
|
361
|
+
if [[ "$BEFORE_DEST" == "$AFTER_DEST" ]]; then
|
|
362
|
+
_pass "user-modified: installed skills dir unchanged across both check invocations (read-only)"
|
|
363
|
+
else
|
|
364
|
+
_fail "user-modified: installed skills dir was modified by the check"
|
|
365
|
+
fi
|
|
366
|
+
if [[ "$BEFORE_KIT_A" == "$AFTER_KIT_A" ]]; then
|
|
367
|
+
_pass "user-modified: kit source A unchanged across both check invocations (read-only)"
|
|
368
|
+
else
|
|
369
|
+
_fail "user-modified: kit source A was modified by the check"
|
|
370
|
+
fi
|
|
371
|
+
|
|
372
|
+
echo ""
|
|
373
|
+
|
|
374
|
+
# ─── Scenario 5: Read-only proof note ────────────────────────────────────────
|
|
375
|
+
# The recursive-checksum before/after assertions inline in Scenarios 1-4 above ARE this
|
|
376
|
+
# scenario's proof — every `skill-drift-check` invocation in this eval (baseline, clean,
|
|
377
|
+
# kit-updated x2, user-modified x2) is bracketed by a directory checksum comparison. No
|
|
378
|
+
# separate invocation is needed here; this section exists so the scenario numbering in this
|
|
379
|
+
# eval's header comment and the plan's Wave 3 task description line up 1:1.
|
|
380
|
+
echo "--- Scenario 5: Read-only proof — asserted inline around every invocation above ---"
|
|
381
|
+
_pass "read-only proof: every check invocation above was bracketed by an unchanged-checksum assertion"
|
|
382
|
+
echo ""
|
|
383
|
+
|
|
384
|
+
# ─── Scenario 6: SessionStart advisory ───────────────────────────────────────
|
|
385
|
+
echo "--- Scenario 6: SessionStart advisory ([SKILL DRIFT] present when drifted, absent when clean) ---"
|
|
386
|
+
|
|
387
|
+
FIXTURE_ROOT="$TMPDIR_EVAL/fixture-repo"
|
|
388
|
+
mkdir -p "$FIXTURE_ROOT/kits" "$FIXTURE_ROOT/dist/claude-code/.claude/skills"
|
|
389
|
+
printf '# Fixture Repo\n' > "$FIXTURE_ROOT/AGENTS.md"
|
|
390
|
+
|
|
391
|
+
SS_DEST="$TMPDIR_EVAL/session-start-dest"
|
|
392
|
+
node -e "
|
|
393
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
394
|
+
const manifest = buildManifest({ skillsSourceDir: '$KIT_A', runtime: 'claude-code' });
|
|
395
|
+
writeManifestAtomic('$SS_DEST/.flow-agents/skills-manifest.json', manifest);
|
|
396
|
+
"
|
|
397
|
+
mkdir -p "$SS_DEST/skills"
|
|
398
|
+
cp -R "$KIT_A/." "$SS_DEST/skills/"
|
|
399
|
+
|
|
400
|
+
# Drifted case: dist bundle reflects KIT_B (skill-one edited vs the installed+manifest
|
|
401
|
+
# baseline) — the same drifted shape proven in Scenario 3.
|
|
402
|
+
rm -rf "$FIXTURE_ROOT/dist/claude-code/.claude/skills"
|
|
403
|
+
cp -R "$KIT_B" "$FIXTURE_ROOT/dist/claude-code/.claude/skills"
|
|
404
|
+
|
|
405
|
+
S6_DRIFTED_OUT="$TMPDIR_EVAL/s6-drifted.out"
|
|
406
|
+
FLOW_AGENTS_USER_CLAUDE_SETTINGS="$SS_DEST/settings.json" node -e "
|
|
407
|
+
const { run } = require('$WORKFLOW_STEERING');
|
|
408
|
+
const out = run(JSON.stringify({ hook_event_name: 'SessionStart', cwd: '$FIXTURE_ROOT' }));
|
|
409
|
+
process.stdout.write(out);
|
|
410
|
+
" > "$S6_DRIFTED_OUT" 2>&1
|
|
411
|
+
|
|
412
|
+
if grep -qF "[SKILL DRIFT]" "$S6_DRIFTED_OUT"; then
|
|
413
|
+
_pass "SessionStart advisory: [SKILL DRIFT] present when the fixture repo is drifted"
|
|
414
|
+
else
|
|
415
|
+
_fail "SessionStart advisory: [SKILL DRIFT] missing for a drifted fixture repo"; cat "$S6_DRIFTED_OUT"
|
|
416
|
+
fi
|
|
417
|
+
|
|
418
|
+
if grep -qF "flow-agents init --runtime claude-code --global" "$S6_DRIFTED_OUT" && grep -qF "flow-agents skill-drift-check" "$S6_DRIFTED_OUT"; then
|
|
419
|
+
_pass "SessionStart advisory: drifted output names the refresh command and points at skill-drift-check"
|
|
420
|
+
else
|
|
421
|
+
_fail "SessionStart advisory: drifted output missing refresh command or skill-drift-check pointer"; cat "$S6_DRIFTED_OUT"
|
|
422
|
+
fi
|
|
423
|
+
|
|
424
|
+
# Clean case: dist bundle reflects KIT_A (matches installed + manifest exactly) -> no drift.
|
|
425
|
+
rm -rf "$FIXTURE_ROOT/dist/claude-code/.claude/skills"
|
|
426
|
+
cp -R "$KIT_A" "$FIXTURE_ROOT/dist/claude-code/.claude/skills"
|
|
427
|
+
|
|
428
|
+
S6_CLEAN_OUT="$TMPDIR_EVAL/s6-clean.out"
|
|
429
|
+
FLOW_AGENTS_USER_CLAUDE_SETTINGS="$SS_DEST/settings.json" node -e "
|
|
430
|
+
const { run } = require('$WORKFLOW_STEERING');
|
|
431
|
+
const out = run(JSON.stringify({ hook_event_name: 'SessionStart', cwd: '$FIXTURE_ROOT' }));
|
|
432
|
+
process.stdout.write(out);
|
|
433
|
+
" > "$S6_CLEAN_OUT" 2>&1
|
|
434
|
+
|
|
435
|
+
if grep -qF "[SKILL DRIFT]" "$S6_CLEAN_OUT"; then
|
|
436
|
+
_fail "SessionStart advisory: [SKILL DRIFT] unexpectedly present for a clean fixture repo"; cat "$S6_CLEAN_OUT"
|
|
437
|
+
else
|
|
438
|
+
_pass "SessionStart advisory: [SKILL DRIFT] absent when the fixture repo is clean"
|
|
439
|
+
fi
|
|
440
|
+
|
|
441
|
+
echo ""
|
|
442
|
+
|
|
443
|
+
# ─── Scenario 7: Unbaselined-only (review fix — closes an eval blind spot) ──
|
|
444
|
+
echo "--- Scenario 7: Unbaselined-only (manifest missing one file's entry) -> exit 1 ---"
|
|
445
|
+
|
|
446
|
+
BASELINE_U="$TMPDIR_EVAL/baseline-u"
|
|
447
|
+
mkdir -p "$BASELINE_U/skill-y"
|
|
448
|
+
printf '# Skill Y\nkit-y-content\n' > "$BASELINE_U/skill-y/SKILL.md"
|
|
449
|
+
|
|
450
|
+
KIT_U="$TMPDIR_EVAL/kit-source-u"
|
|
451
|
+
mkdir -p "$KIT_U/skill-x" "$KIT_U/skill-y"
|
|
452
|
+
printf '# Skill X\nkit-x-v2\n' > "$KIT_U/skill-x/SKILL.md"
|
|
453
|
+
printf '# Skill Y\nkit-y-content\n' > "$KIT_U/skill-y/SKILL.md"
|
|
454
|
+
|
|
455
|
+
DEST_U="$TMPDIR_EVAL/dest-u"
|
|
456
|
+
mkdir -p "$DEST_U/sk""ills/skill-x" "$DEST_U/sk""ills/skill-y"
|
|
457
|
+
printf '# Skill X\ninstalled-x-old\n' > "$DEST_U/sk""ills/skill-x/SKILL.md"
|
|
458
|
+
printf '# Skill Y\nkit-y-content\n' > "$DEST_U/sk""ills/skill-y/SKILL.md"
|
|
459
|
+
|
|
460
|
+
# Manifest is built from BASELINE_U (skill-y only) — skill-x deliberately has NO manifest entry
|
|
461
|
+
# at all, simulating "manifest present but missing one file's entry".
|
|
462
|
+
node -e "
|
|
463
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
464
|
+
const manifest = buildManifest({ skillsSourceDir: '$BASELINE_U', runtime: 'claude-code' });
|
|
465
|
+
writeManifestAtomic('$DEST_U/.flow-agents/skills-manifest.json', manifest);
|
|
466
|
+
"
|
|
467
|
+
|
|
468
|
+
BEFORE_KIT_U=$(_dirsum "$KIT_U")
|
|
469
|
+
BEFORE_DEST_U=$(_dirsum "$DEST_U/skills")
|
|
470
|
+
S7_OUT="$TMPDIR_EVAL/s7.out"
|
|
471
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_U" --dest "$DEST_U" > "$S7_OUT" 2>&1; then
|
|
472
|
+
S7_RC=0
|
|
473
|
+
else
|
|
474
|
+
S7_RC=$?
|
|
475
|
+
fi
|
|
476
|
+
S7_JSON="$TMPDIR_EVAL/s7.json"
|
|
477
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_U" --dest "$DEST_U" --json > "$S7_JSON" 2>&1; then
|
|
478
|
+
S7_JSON_RC=0
|
|
479
|
+
else
|
|
480
|
+
S7_JSON_RC=$?
|
|
481
|
+
fi
|
|
482
|
+
AFTER_KIT_U=$(_dirsum "$KIT_U")
|
|
483
|
+
AFTER_DEST_U=$(_dirsum "$DEST_U/skills")
|
|
484
|
+
|
|
485
|
+
if [[ "$S7_RC" -eq 1 && "$S7_JSON_RC" -eq 1 ]]; then
|
|
486
|
+
_pass "unbaselined-only: both plain and --json runs exit 1"
|
|
487
|
+
else
|
|
488
|
+
_fail "unbaselined-only: expected exit 1/1, got plain=$S7_RC json=$S7_JSON_RC"; cat "$S7_OUT"; cat "$S7_JSON"
|
|
489
|
+
fi
|
|
490
|
+
|
|
491
|
+
if grep -q "unbaselined" "$S7_OUT" && grep -q "skill-x/SKILL.md" "$S7_OUT"; then
|
|
492
|
+
_pass "unbaselined-only: plain output lists skill-x/SKILL.md under unbaselined"
|
|
493
|
+
else
|
|
494
|
+
_fail "unbaselined-only: plain output did not list the file under unbaselined"; cat "$S7_OUT"
|
|
495
|
+
fi
|
|
496
|
+
|
|
497
|
+
if node - "$S7_JSON" << 'NODE'
|
|
498
|
+
const fs = require("node:fs");
|
|
499
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
500
|
+
if (report.hasDrift !== true) throw new Error("hasDrift should be true: " + JSON.stringify(report.summary));
|
|
501
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
502
|
+
if (byPath["skill-x/SKILL.md"] !== "unbaselined") throw new Error("skill-x/SKILL.md should be unbaselined, got " + byPath["skill-x/SKILL.md"]);
|
|
503
|
+
if (byPath["skill-y/SKILL.md"] !== "in_sync") throw new Error("skill-y/SKILL.md should remain in_sync, got " + byPath["skill-y/SKILL.md"]);
|
|
504
|
+
console.log("ok");
|
|
505
|
+
NODE
|
|
506
|
+
then
|
|
507
|
+
_pass "unbaselined-only: --json report classifies exactly skill-x/SKILL.md as unbaselined, others in_sync"
|
|
508
|
+
else
|
|
509
|
+
_fail "unbaselined-only: --json report classification incorrect"
|
|
510
|
+
fi
|
|
511
|
+
|
|
512
|
+
if [[ "$BEFORE_KIT_U" == "$AFTER_KIT_U" && "$BEFORE_DEST_U" == "$AFTER_DEST_U" ]]; then
|
|
513
|
+
_pass "unbaselined-only: kit source and installed dir unchanged across both check invocations (read-only)"
|
|
514
|
+
else
|
|
515
|
+
_fail "unbaselined-only: kit source or installed dir was modified by the check"
|
|
516
|
+
fi
|
|
517
|
+
|
|
518
|
+
echo ""
|
|
519
|
+
|
|
520
|
+
# ─── Scenario 8: Missing-install-only (review fix) ───────────────────────────
|
|
521
|
+
echo "--- Scenario 8: Missing-install-only (kit file, no installed counterpart) -> exit 1 ---"
|
|
522
|
+
|
|
523
|
+
KIT_M="$TMPDIR_EVAL/kit-source-m"
|
|
524
|
+
mkdir -p "$KIT_M/skill-p" "$KIT_M/skill-q"
|
|
525
|
+
printf '# Skill P\np-content\n' > "$KIT_M/skill-p/SKILL.md"
|
|
526
|
+
printf '# Skill Q\nq-content\n' > "$KIT_M/skill-q/SKILL.md"
|
|
527
|
+
|
|
528
|
+
DEST_M="$TMPDIR_EVAL/dest-m"
|
|
529
|
+
mkdir -p "$DEST_M/sk""ills/skill-p"
|
|
530
|
+
printf '# Skill P\np-content\n' > "$DEST_M/sk""ills/skill-p/SKILL.md"
|
|
531
|
+
# skill-q is deliberately never installed at all.
|
|
532
|
+
|
|
533
|
+
# Manifest entry for skill-q is optional and does not affect classification (missing_install is
|
|
534
|
+
# determined purely by kitHash !== null && installedHash === null) — built from the installed
|
|
535
|
+
# dir itself (skill-p only) to prove that.
|
|
536
|
+
node -e "
|
|
537
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
538
|
+
const manifest = buildManifest({ skillsSourceDir: '$DEST_M/skills', runtime: 'claude-code' });
|
|
539
|
+
writeManifestAtomic('$DEST_M/.flow-agents/skills-manifest.json', manifest);
|
|
540
|
+
"
|
|
541
|
+
|
|
542
|
+
BEFORE_KIT_M=$(_dirsum "$KIT_M")
|
|
543
|
+
BEFORE_DEST_M=$(_dirsum "$DEST_M/skills")
|
|
544
|
+
S8_OUT="$TMPDIR_EVAL/s8.out"
|
|
545
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_M" --dest "$DEST_M" > "$S8_OUT" 2>&1; then
|
|
546
|
+
S8_RC=0
|
|
547
|
+
else
|
|
548
|
+
S8_RC=$?
|
|
549
|
+
fi
|
|
550
|
+
S8_JSON="$TMPDIR_EVAL/s8.json"
|
|
551
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_M" --dest "$DEST_M" --json > "$S8_JSON" 2>&1; then
|
|
552
|
+
S8_JSON_RC=0
|
|
553
|
+
else
|
|
554
|
+
S8_JSON_RC=$?
|
|
555
|
+
fi
|
|
556
|
+
AFTER_KIT_M=$(_dirsum "$KIT_M")
|
|
557
|
+
AFTER_DEST_M=$(_dirsum "$DEST_M/skills")
|
|
558
|
+
|
|
559
|
+
if [[ "$S8_RC" -eq 1 && "$S8_JSON_RC" -eq 1 ]]; then
|
|
560
|
+
_pass "missing-install-only: both plain and --json runs exit 1"
|
|
561
|
+
else
|
|
562
|
+
_fail "missing-install-only: expected exit 1/1, got plain=$S8_RC json=$S8_JSON_RC"; cat "$S8_OUT"; cat "$S8_JSON"
|
|
563
|
+
fi
|
|
564
|
+
|
|
565
|
+
if grep -q "missing_install" "$S8_OUT" && grep -q "skill-q/SKILL.md" "$S8_OUT"; then
|
|
566
|
+
_pass "missing-install-only: plain output lists skill-q/SKILL.md under missing_install"
|
|
567
|
+
else
|
|
568
|
+
_fail "missing-install-only: plain output did not list the file under missing_install"; cat "$S8_OUT"
|
|
569
|
+
fi
|
|
570
|
+
|
|
571
|
+
if node - "$S8_JSON" << 'NODE'
|
|
572
|
+
const fs = require("node:fs");
|
|
573
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
574
|
+
if (report.hasDrift !== true) throw new Error("hasDrift should be true: " + JSON.stringify(report.summary));
|
|
575
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
576
|
+
if (byPath["skill-q/SKILL.md"] !== "missing_install") throw new Error("skill-q/SKILL.md should be missing_install, got " + byPath["skill-q/SKILL.md"]);
|
|
577
|
+
if (byPath["skill-p/SKILL.md"] !== "in_sync") throw new Error("skill-p/SKILL.md should remain in_sync, got " + byPath["skill-p/SKILL.md"]);
|
|
578
|
+
console.log("ok");
|
|
579
|
+
NODE
|
|
580
|
+
then
|
|
581
|
+
_pass "missing-install-only: --json report classifies exactly skill-q/SKILL.md as missing_install, others in_sync"
|
|
582
|
+
else
|
|
583
|
+
_fail "missing-install-only: --json report classification incorrect"
|
|
584
|
+
fi
|
|
585
|
+
|
|
586
|
+
if [[ "$BEFORE_KIT_M" == "$AFTER_KIT_M" && "$BEFORE_DEST_M" == "$AFTER_DEST_M" ]]; then
|
|
587
|
+
_pass "missing-install-only: kit source and installed dir unchanged across both check invocations (read-only)"
|
|
588
|
+
else
|
|
589
|
+
_fail "missing-install-only: kit source or installed dir was modified by the check"
|
|
590
|
+
fi
|
|
591
|
+
|
|
592
|
+
echo ""
|
|
593
|
+
|
|
594
|
+
# ─── Scenario 9: Kit-removed (review fix — new distinct drift state) ────────
|
|
595
|
+
echo "--- Scenario 9: Kit-removed (installed file removed from current kit source) -> exit 1 ---"
|
|
596
|
+
|
|
597
|
+
BASELINE_R="$TMPDIR_EVAL/baseline-r"
|
|
598
|
+
mkdir -p "$BASELINE_R/skill-r" "$BASELINE_R/skill-s"
|
|
599
|
+
printf '# Skill R\nr-content-v1\n' > "$BASELINE_R/skill-r/SKILL.md"
|
|
600
|
+
printf '# Skill S\ns-content\n' > "$BASELINE_R/skill-s/SKILL.md"
|
|
601
|
+
|
|
602
|
+
DEST_R="$TMPDIR_EVAL/dest-r"
|
|
603
|
+
mkdir -p "$DEST_R/sk""ills/skill-r" "$DEST_R/sk""ills/skill-s"
|
|
604
|
+
printf '# Skill R\nr-content-v1\n' > "$DEST_R/sk""ills/skill-r/SKILL.md"
|
|
605
|
+
printf '# Skill S\ns-content\n' > "$DEST_R/sk""ills/skill-s/SKILL.md"
|
|
606
|
+
|
|
607
|
+
# Manifest baseline records BOTH skill-r and skill-s.
|
|
608
|
+
node -e "
|
|
609
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
610
|
+
const manifest = buildManifest({ skillsSourceDir: '$BASELINE_R', runtime: 'claude-code' });
|
|
611
|
+
writeManifestAtomic('$DEST_R/.flow-agents/skills-manifest.json', manifest);
|
|
612
|
+
"
|
|
613
|
+
|
|
614
|
+
# Current kit source no longer ships skill-r at all (removed upstream) — skill-s unchanged.
|
|
615
|
+
KIT_R="$TMPDIR_EVAL/kit-source-r"
|
|
616
|
+
mkdir -p "$KIT_R/skill-s"
|
|
617
|
+
printf '# Skill S\ns-content\n' > "$KIT_R/skill-s/SKILL.md"
|
|
618
|
+
|
|
619
|
+
BEFORE_KIT_R=$(_dirsum "$KIT_R")
|
|
620
|
+
BEFORE_DEST_R=$(_dirsum "$DEST_R/skills")
|
|
621
|
+
S9_OUT="$TMPDIR_EVAL/s9.out"
|
|
622
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_R" --dest "$DEST_R" > "$S9_OUT" 2>&1; then
|
|
623
|
+
S9_RC=0
|
|
624
|
+
else
|
|
625
|
+
S9_RC=$?
|
|
626
|
+
fi
|
|
627
|
+
S9_JSON="$TMPDIR_EVAL/s9.json"
|
|
628
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_R" --dest "$DEST_R" --json > "$S9_JSON" 2>&1; then
|
|
629
|
+
S9_JSON_RC=0
|
|
630
|
+
else
|
|
631
|
+
S9_JSON_RC=$?
|
|
632
|
+
fi
|
|
633
|
+
AFTER_KIT_R=$(_dirsum "$KIT_R")
|
|
634
|
+
AFTER_DEST_R=$(_dirsum "$DEST_R/skills")
|
|
635
|
+
|
|
636
|
+
if [[ "$S9_RC" -eq 1 && "$S9_JSON_RC" -eq 1 ]]; then
|
|
637
|
+
_pass "kit-removed: both plain and --json runs exit 1"
|
|
638
|
+
else
|
|
639
|
+
_fail "kit-removed: expected exit 1/1, got plain=$S9_RC json=$S9_JSON_RC"; cat "$S9_OUT"; cat "$S9_JSON"
|
|
640
|
+
fi
|
|
641
|
+
|
|
642
|
+
if grep -q "kit_removed" "$S9_OUT" && grep -q "skill-r/SKILL.md" "$S9_OUT"; then
|
|
643
|
+
_pass "kit-removed: plain output lists skill-r/SKILL.md under kit_removed"
|
|
644
|
+
else
|
|
645
|
+
_fail "kit-removed: plain output did not list the file under kit_removed"; cat "$S9_OUT"
|
|
646
|
+
fi
|
|
647
|
+
|
|
648
|
+
if grep -qF "refresh will NOT delete them" "$S9_OUT" && grep -qF "review and remove them manually" "$S9_OUT"; then
|
|
649
|
+
_pass "kit-removed: plain output carries the accurate 'refresh will NOT delete it' guidance"
|
|
650
|
+
else
|
|
651
|
+
_fail "kit-removed: plain output missing the kit_removed-specific guidance text"; cat "$S9_OUT"
|
|
652
|
+
fi
|
|
653
|
+
|
|
654
|
+
if node - "$S9_JSON" << 'NODE'
|
|
655
|
+
const fs = require("node:fs");
|
|
656
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
657
|
+
if (report.hasDrift !== true) throw new Error("hasDrift should be true: " + JSON.stringify(report.summary));
|
|
658
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
659
|
+
if (byPath["skill-r/SKILL.md"] !== "kit_removed") throw new Error("skill-r/SKILL.md should be kit_removed, got " + byPath["skill-r/SKILL.md"]);
|
|
660
|
+
if (byPath["skill-s/SKILL.md"] !== "in_sync") throw new Error("skill-s/SKILL.md should remain in_sync, got " + byPath["skill-s/SKILL.md"]);
|
|
661
|
+
if (report.summary.kitRemoved !== 1) throw new Error("expected summary.kitRemoved === 1, got " + report.summary.kitRemoved);
|
|
662
|
+
console.log("ok");
|
|
663
|
+
NODE
|
|
664
|
+
then
|
|
665
|
+
_pass "kit-removed: --json report classifies exactly skill-r/SKILL.md as kit_removed (summary.kitRemoved === 1), skill-s in_sync"
|
|
666
|
+
else
|
|
667
|
+
_fail "kit-removed: --json report classification incorrect"
|
|
668
|
+
fi
|
|
669
|
+
|
|
670
|
+
if [[ "$BEFORE_KIT_R" == "$AFTER_KIT_R" && "$BEFORE_DEST_R" == "$AFTER_DEST_R" ]]; then
|
|
671
|
+
_pass "kit-removed: kit source and installed dir unchanged across both check invocations (read-only)"
|
|
672
|
+
else
|
|
673
|
+
_fail "kit-removed: kit source or installed dir was modified by the check"
|
|
674
|
+
fi
|
|
675
|
+
|
|
676
|
+
echo ""
|
|
677
|
+
|
|
678
|
+
# ─── Scenario 10: Corrupt-manifest SessionStart advisory (review fix) ───────
|
|
679
|
+
echo "--- Scenario 10: Corrupt-manifest + clean install -> SessionStart advisory does not crash, no drift line ---"
|
|
680
|
+
|
|
681
|
+
FIXTURE_ROOT2="$TMPDIR_EVAL/fixture-repo-corrupt"
|
|
682
|
+
mkdir -p "$FIXTURE_ROOT2/kits" "$FIXTURE_ROOT2/dist/claude-code/.claude/skills"
|
|
683
|
+
printf '# Fixture Repo\n' > "$FIXTURE_ROOT2/AGENTS.md"
|
|
684
|
+
rm -rf "$FIXTURE_ROOT2/dist/claude-code/.claude/skills"
|
|
685
|
+
cp -R "$KIT_A" "$FIXTURE_ROOT2/dist/claude-code/.claude/skills"
|
|
686
|
+
|
|
687
|
+
SS_DEST2="$TMPDIR_EVAL/session-start-dest-corrupt"
|
|
688
|
+
mkdir -p "$SS_DEST2/skills" "$SS_DEST2/.flow-agents"
|
|
689
|
+
cp -R "$KIT_A/." "$SS_DEST2/skills/"
|
|
690
|
+
# Invalid JSON manifest — loadManifest() must tolerate this as "absent" (null), never throw.
|
|
691
|
+
printf '{ this is not valid json' > "$SS_DEST2/.flow-agents/skills-manifest.json"
|
|
692
|
+
|
|
693
|
+
S10_OUT="$TMPDIR_EVAL/s10.out"
|
|
694
|
+
FLOW_AGENTS_USER_CLAUDE_SETTINGS="$SS_DEST2/settings.json" node -e "
|
|
695
|
+
const { run } = require('$WORKFLOW_STEERING');
|
|
696
|
+
const out = run(JSON.stringify({ hook_event_name: 'SessionStart', cwd: '$FIXTURE_ROOT2' }));
|
|
697
|
+
process.stdout.write(out);
|
|
698
|
+
" > "$S10_OUT" 2>&1
|
|
699
|
+
S10_RC=$?
|
|
700
|
+
|
|
701
|
+
if [[ "$S10_RC" -eq 0 ]]; then
|
|
702
|
+
_pass "corrupt-manifest SessionStart: advisory invocation exits 0 (no crash)"
|
|
703
|
+
else
|
|
704
|
+
_fail "corrupt-manifest SessionStart: advisory invocation crashed (exit $S10_RC)"; cat "$S10_OUT"
|
|
705
|
+
fi
|
|
706
|
+
|
|
707
|
+
if grep -qF "[SKILL DRIFT]" "$S10_OUT"; then
|
|
708
|
+
_fail "corrupt-manifest SessionStart: [SKILL DRIFT] unexpectedly present despite a clean install"; cat "$S10_OUT"
|
|
709
|
+
else
|
|
710
|
+
_pass "corrupt-manifest SessionStart: [SKILL DRIFT] absent (corrupt manifest tolerated as absent, install is clean)"
|
|
711
|
+
fi
|
|
712
|
+
|
|
713
|
+
echo ""
|
|
714
|
+
|
|
715
|
+
# --- Scenario 11: Foreign (non-kit) installed file -- outside jurisdiction, must be silently
|
|
716
|
+
# skipped (kontourai/flow-agents#465). A real machine's shared installed-skills dest can hold
|
|
717
|
+
# other tools' skill directories the kit never shipped and never recorded in any manifest --
|
|
718
|
+
# these must never surface as "unbaselined" (that state is reserved for kit-owned paths that
|
|
719
|
+
# simply predate the manifest baseline).
|
|
720
|
+
echo "--- Scenario 11: Foreign installed file (no kit-source or manifest entry) -> ignored ---"
|
|
721
|
+
|
|
722
|
+
DEST_F="$TMPDIR_EVAL/dest-f"
|
|
723
|
+
mkdir -p "$DEST_F/sk""ills"
|
|
724
|
+
cp -R "$KIT_A/." "$DEST_F/sk""ills/"
|
|
725
|
+
mkdir -p "$DEST_F/sk""ills/other-tool-sk""ill"
|
|
726
|
+
printf '# Other Tool Skill\nnever shipped by this kit, never in any manifest\n' > "$DEST_F/sk""ills/other-tool-sk""ill/SKILL.md"
|
|
727
|
+
|
|
728
|
+
# Manifest baseline built from KIT_A only -- the foreign file was never part of any kit source,
|
|
729
|
+
# so it (correctly) never gets a manifest entry either.
|
|
730
|
+
node -e "
|
|
731
|
+
const { buildManifest, writeManifestAtomic } = require('$SKILL_DRIFT_LIB');
|
|
732
|
+
const manifest = buildManifest({ skillsSourceDir: '$KIT_A', runtime: 'claude-code' });
|
|
733
|
+
writeManifestAtomic('$DEST_F/.flow-agents/skills-manifest.json', manifest);
|
|
734
|
+
"
|
|
735
|
+
|
|
736
|
+
# --- 11a: kit files in sync, foreign file present -> exit 0, foreign path absent from output ---
|
|
737
|
+
BEFORE_DEST_F=$(_dirsum "$DEST_F/sk""ills")
|
|
738
|
+
BEFORE_KIT_A=$(_dirsum "$KIT_A")
|
|
739
|
+
S11A_OUT="$TMPDIR_EVAL/s11a.out"
|
|
740
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST_F" > "$S11A_OUT" 2>&1; then
|
|
741
|
+
S11A_RC=0
|
|
742
|
+
else
|
|
743
|
+
S11A_RC=$?
|
|
744
|
+
fi
|
|
745
|
+
S11A_JSON="$TMPDIR_EVAL/s11a.json"
|
|
746
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_A" --dest "$DEST_F" --json > "$S11A_JSON" 2>&1; then
|
|
747
|
+
S11A_JSON_RC=0
|
|
748
|
+
else
|
|
749
|
+
S11A_JSON_RC=$?
|
|
750
|
+
fi
|
|
751
|
+
AFTER_DEST_F=$(_dirsum "$DEST_F/sk""ills")
|
|
752
|
+
AFTER_KIT_A=$(_dirsum "$KIT_A")
|
|
753
|
+
|
|
754
|
+
if [[ "$S11A_RC" -eq 0 && "$S11A_JSON_RC" -eq 0 ]]; then
|
|
755
|
+
_pass "foreign-file clean: both plain and --json runs exit 0 despite foreign file present"
|
|
756
|
+
else
|
|
757
|
+
_fail "foreign-file clean: expected exit 0/0, got plain=$S11A_RC json=$S11A_JSON_RC"; cat "$S11A_OUT"; cat "$S11A_JSON"
|
|
758
|
+
fi
|
|
759
|
+
|
|
760
|
+
if grep -q "other-tool-sk""ill" "$S11A_OUT"; then
|
|
761
|
+
_fail "foreign-file clean: plain output unexpectedly mentions the foreign file"; cat "$S11A_OUT"
|
|
762
|
+
else
|
|
763
|
+
_pass "foreign-file clean: plain output never mentions the foreign file"
|
|
764
|
+
fi
|
|
765
|
+
|
|
766
|
+
if node - "$S11A_JSON" << 'NODE'
|
|
767
|
+
const fs = require("node:fs");
|
|
768
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
769
|
+
if (report.hasDrift !== false) throw new Error("hasDrift should be false: " + JSON.stringify(report.summary));
|
|
770
|
+
if (report.summary.total !== 3) throw new Error("expected 3 files total (foreign file excluded), got " + report.summary.total);
|
|
771
|
+
if (report.files.some((f) => f.path.includes("other-tool"))) throw new Error("foreign file leaked into files[]: " + JSON.stringify(report.files));
|
|
772
|
+
console.log("ok");
|
|
773
|
+
NODE
|
|
774
|
+
then
|
|
775
|
+
_pass "foreign-file clean: --json report excludes the foreign file from files[]/summary, hasDrift=false"
|
|
776
|
+
else
|
|
777
|
+
_fail "foreign-file clean: --json report leaked the foreign file or misreported hasDrift"
|
|
778
|
+
fi
|
|
779
|
+
|
|
780
|
+
if [[ "$BEFORE_DEST_F" == "$AFTER_DEST_F" && "$BEFORE_KIT_A" == "$AFTER_KIT_A" ]]; then
|
|
781
|
+
_pass "foreign-file clean: installed dir (incl. foreign file) and kit source unchanged (read-only)"
|
|
782
|
+
else
|
|
783
|
+
_fail "foreign-file clean: installed dir or kit source was modified by the check"
|
|
784
|
+
fi
|
|
785
|
+
|
|
786
|
+
echo ""
|
|
787
|
+
|
|
788
|
+
# --- 11b: one drifted kit file, foreign file still present -> exit 1 listing only the kit file ---
|
|
789
|
+
echo "--- Scenario 11b: Foreign file + one drifted kit file -> exit 1, foreign file still ignored ---"
|
|
790
|
+
|
|
791
|
+
BEFORE_DEST_F=$(_dirsum "$DEST_F/sk""ills")
|
|
792
|
+
BEFORE_KIT_B=$(_dirsum "$KIT_B")
|
|
793
|
+
S11B_OUT="$TMPDIR_EVAL/s11b.out"
|
|
794
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_B" --dest "$DEST_F" > "$S11B_OUT" 2>&1; then
|
|
795
|
+
S11B_RC=0
|
|
796
|
+
else
|
|
797
|
+
S11B_RC=$?
|
|
798
|
+
fi
|
|
799
|
+
S11B_JSON="$TMPDIR_EVAL/s11b.json"
|
|
800
|
+
if node "$CLI" skill-drift-check --kit-source-dir "$KIT_B" --dest "$DEST_F" --json > "$S11B_JSON" 2>&1; then
|
|
801
|
+
S11B_JSON_RC=0
|
|
802
|
+
else
|
|
803
|
+
S11B_JSON_RC=$?
|
|
804
|
+
fi
|
|
805
|
+
AFTER_DEST_F=$(_dirsum "$DEST_F/sk""ills")
|
|
806
|
+
AFTER_KIT_B=$(_dirsum "$KIT_B")
|
|
807
|
+
|
|
808
|
+
if [[ "$S11B_RC" -eq 1 && "$S11B_JSON_RC" -eq 1 ]]; then
|
|
809
|
+
_pass "foreign-file + kit-updated: both plain and --json runs exit 1"
|
|
810
|
+
else
|
|
811
|
+
_fail "foreign-file + kit-updated: expected exit 1/1, got plain=$S11B_RC json=$S11B_JSON_RC"; cat "$S11B_OUT"; cat "$S11B_JSON"
|
|
812
|
+
fi
|
|
813
|
+
|
|
814
|
+
if grep -q "kit_updated" "$S11B_OUT" && grep -q "skill-one/SKILL.md" "$S11B_OUT" && ! grep -q "other-tool-sk""ill" "$S11B_OUT"; then
|
|
815
|
+
_pass "foreign-file + kit-updated: plain output lists only skill-one/SKILL.md under kit_updated, never the foreign file"
|
|
816
|
+
else
|
|
817
|
+
_fail "foreign-file + kit-updated: plain output incorrect"; cat "$S11B_OUT"
|
|
818
|
+
fi
|
|
819
|
+
|
|
820
|
+
if node - "$S11B_JSON" << 'NODE'
|
|
821
|
+
const fs = require("node:fs");
|
|
822
|
+
const report = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
823
|
+
if (report.hasDrift !== true) throw new Error("hasDrift should be true: " + JSON.stringify(report.summary));
|
|
824
|
+
if (report.summary.total !== 3) throw new Error("expected 3 files total (foreign file excluded), got " + report.summary.total);
|
|
825
|
+
const byPath = Object.fromEntries(report.files.map((f) => [f.path, f.state]));
|
|
826
|
+
if (byPath["skill-one/SKILL.md"] !== "kit_updated") throw new Error("skill-one/SKILL.md should be kit_updated, got " + byPath["skill-one/SKILL.md"]);
|
|
827
|
+
if (report.files.some((f) => f.path.includes("other-tool"))) throw new Error("foreign file leaked into files[]: " + JSON.stringify(report.files));
|
|
828
|
+
console.log("ok");
|
|
829
|
+
NODE
|
|
830
|
+
then
|
|
831
|
+
_pass "foreign-file + kit-updated: --json report lists only the kit file, foreign file excluded"
|
|
832
|
+
else
|
|
833
|
+
_fail "foreign-file + kit-updated: --json report classification incorrect"
|
|
834
|
+
fi
|
|
835
|
+
|
|
836
|
+
if [[ "$BEFORE_DEST_F" == "$AFTER_DEST_F" && "$BEFORE_KIT_B" == "$AFTER_KIT_B" ]]; then
|
|
837
|
+
_pass "foreign-file + kit-updated: installed dir (incl. foreign file) and kit source unchanged (read-only)"
|
|
838
|
+
else
|
|
839
|
+
_fail "foreign-file + kit-updated: installed dir or kit source was modified by the check"
|
|
840
|
+
fi
|
|
841
|
+
|
|
842
|
+
echo ""
|
|
843
|
+
|
|
844
|
+
# --- 11c: SessionStart advisory, foreign file present + kit in sync -> no [SKILL DRIFT] line ---
|
|
845
|
+
echo "--- Scenario 11c: SessionStart advisory, foreign file + in-sync kit -> no [SKILL DRIFT] ---"
|
|
846
|
+
|
|
847
|
+
FIXTURE_ROOT3="$TMPDIR_EVAL/fixture-repo-foreign"
|
|
848
|
+
mkdir -p "$FIXTURE_ROOT3/kits" "$FIXTURE_ROOT3/dist/claude-code/.claude"
|
|
849
|
+
printf '# Fixture Repo\n' > "$FIXTURE_ROOT3/AGENTS.md"
|
|
850
|
+
cp -R "$KIT_A" "$FIXTURE_ROOT3/dist/claude-code/.claude/sk""ills"
|
|
851
|
+
|
|
852
|
+
S11C_OUT="$TMPDIR_EVAL/s11c.out"
|
|
853
|
+
FLOW_AGENTS_USER_CLAUDE_SETTINGS="$DEST_F/settings.json" node -e "
|
|
854
|
+
const { run } = require('$WORKFLOW_STEERING');
|
|
855
|
+
const out = run(JSON.stringify({ hook_event_name: 'SessionStart', cwd: '$FIXTURE_ROOT3' }));
|
|
856
|
+
process.stdout.write(out);
|
|
857
|
+
" > "$S11C_OUT" 2>&1
|
|
858
|
+
|
|
859
|
+
if grep -qF "[SKILL DRIFT]" "$S11C_OUT"; then
|
|
860
|
+
_fail "foreign-file SessionStart: [SKILL DRIFT] unexpectedly present despite an in-sync kit"; cat "$S11C_OUT"
|
|
861
|
+
else
|
|
862
|
+
_pass "foreign-file SessionStart: [SKILL DRIFT] absent when kit is in sync despite foreign file present"
|
|
863
|
+
fi
|
|
864
|
+
|
|
865
|
+
echo ""
|
|
866
|
+
echo "==========================="
|
|
867
|
+
total=$((pass + fail))
|
|
868
|
+
echo "Results: ${pass}/${total} passed, ${fail} failed"
|
|
869
|
+
[[ "$fail" -gt 0 ]] && exit 1
|
|
870
|
+
exit 0
|