@hassangameryt/freeflow 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +2 -2
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +2 -2
- package/.codex-plugin/plugin.json +3 -3
- package/.cursor-plugin/plugin.json +1 -1
- package/README.md +96 -149
- package/capabilities/cognitive-routing/SKILL.md +21 -129
- package/capabilities/cognitive-routing/references/coordinator.md +163 -0
- package/capabilities/cognitive-routing/references/worker.md +38 -0
- package/capabilities/compaction/SKILL.md +3 -3
- package/capabilities/compaction/references/summary-format.md +26 -15
- package/capabilities/messaging/SKILL.md +49 -0
- package/capabilities/task-memory/SKILL.md +87 -0
- package/command-surface.json +6 -54
- package/gemini-extension.json +1 -1
- package/hooks/claude/hooks.json +13 -3
- package/hooks/codex/hooks.json +2 -1
- package/hooks/shared/runtime-context.mjs +69 -77
- package/package.json +25 -8
- package/pi-extension/dist/ask-user/ask.js +108 -0
- package/pi-extension/dist/ask-user/form.js +250 -0
- package/pi-extension/dist/ask-user/questions.js +66 -0
- package/pi-extension/dist/ask-user/tool.js +124 -0
- package/pi-extension/dist/cognitive-routing/assembler.js +35 -11
- package/pi-extension/dist/cognitive-routing/bootstrap.js +90 -0
- package/pi-extension/dist/cognitive-routing/config.js +3 -4
- package/pi-extension/dist/cognitive-routing/event-store.js +4 -1
- package/pi-extension/dist/cognitive-routing/gate.js +20 -5
- package/pi-extension/dist/cognitive-routing/handlers.js +15 -13
- package/pi-extension/dist/cognitive-routing/model-control.js +17 -8
- package/pi-extension/dist/cognitive-routing/projection.js +23 -6
- package/pi-extension/dist/cognitive-routing/provenance.js +3 -1
- package/pi-extension/dist/cognitive-routing/render.js +15 -15
- package/pi-extension/dist/cognitive-routing/runtime.js +4 -2
- package/pi-extension/dist/cognitive-routing/schemas.js +4 -4
- package/pi-extension/dist/cognitive-routing/session.js +3 -1
- package/pi-extension/dist/cognitive-routing/sources.js +29 -4
- package/pi-extension/dist/cognitive-routing/state.js +18 -8
- package/pi-extension/dist/cognitive-routing/status.js +4 -3
- package/pi-extension/dist/cognitive-routing/tools.js +29 -15
- package/pi-extension/dist/cognitive-routing/types.js +10 -1
- package/pi-extension/dist/compaction/carry.js +36 -22
- package/pi-extension/dist/compaction/config.js +4 -2
- package/pi-extension/dist/compaction/controller.js +52 -29
- package/pi-extension/dist/compaction/harness.js +19 -3
- package/pi-extension/dist/compaction/results.js +14 -13
- package/pi-extension/dist/compaction/tool.js +6 -6
- package/pi-extension/dist/host/catalog.js +20 -35
- package/pi-extension/dist/host/config-validation.js +233 -0
- package/pi-extension/dist/host/config.js +78 -91
- package/pi-extension/dist/host/inbound.js +142 -0
- package/pi-extension/dist/host/projection-tags.js +1 -1
- package/pi-extension/dist/host/prompts.js +24 -13
- package/pi-extension/dist/host/read-only-session.js +6 -1
- package/pi-extension/dist/host/request-history.js +7 -3
- package/pi-extension/dist/host/rows.js +25 -0
- package/pi-extension/dist/host/runtime-state.js +17 -2
- package/pi-extension/dist/host/section-keeper.js +85 -0
- package/pi-extension/dist/host/settings/freeflow-command.js +846 -93
- package/pi-extension/dist/host/settings/settings-view.js +27 -8
- package/pi-extension/dist/host/status.js +19 -5
- package/pi-extension/dist/index.js +638 -72
- package/pi-extension/dist/messaging/config.js +27 -0
- package/pi-extension/dist/messaging/messaging.js +398 -0
- package/pi-extension/dist/messaging/registry.js +57 -0
- package/pi-extension/dist/messaging/render.js +90 -0
- package/pi-extension/dist/messaging/tools.js +109 -0
- package/pi-extension/dist/messaging/transport.js +101 -0
- package/pi-extension/dist/provider-support/cache/keepalive.js +16 -4
- package/pi-extension/dist/task-memory/config.js +49 -0
- package/pi-extension/dist/task-memory/registry.js +119 -0
- package/pi-extension/dist/task-memory/render.js +107 -0
- package/pi-extension/dist/task-memory/runtime.js +994 -0
- package/pi-extension/dist/task-memory/tool.js +233 -0
- package/pi-extension/dist/tool-execution/apply-patch/tool.js +1 -1
- package/pi-extension/dist/tool-execution/background.js +220 -28
- package/pi-extension/dist/tool-execution/config.js +14 -3
- package/pi-extension/dist/tool-execution/file-tracking.js +24 -4
- package/pi-extension/dist/tool-execution/guidance-notices.js +124 -0
- package/pi-extension/dist/tool-execution/skill-reads.js +143 -0
- package/pi-extension/dist/tool-execution/tools.js +2 -2
- package/runtime/prompts/cognitive-routing.md +4 -4
- package/runtime/prompts/core.md +24 -14
- package/runtime/prompts/interaction-contract.md +7 -4
- package/runtime/prompts/tool-execution.md +3 -16
- package/runtime/prompts/working-method.md +16 -99
- package/skills/deliver-work/SKILL.md +76 -0
- package/skills/deliver-work/references/branch-closeout.md +134 -0
- package/skills/deliver-work/references/commit.md +83 -0
- package/skills/deliver-work/references/launch.md +94 -0
- package/skills/deliver-work/references/release.md +85 -0
- package/skills/deliver-work/scripts/commit-state.mjs +139 -0
- package/skills/deliver-work/scripts/delivery-state.mjs +107 -0
- package/skills/deliver-work/scripts/lib/git-state.mjs +178 -0
- package/skills/deliver-work/scripts/push-route.mjs +122 -0
- package/skills/deliver-work/scripts/release-preflight.mjs +177 -0
- package/skills/discuss-work/SKILL.md +148 -0
- package/skills/{discuss → discuss-work}/references/discussion-continuity.md +10 -27
- package/skills/discuss-work/references/learning-action.md +36 -0
- package/skills/discuss-work/scripts/choices.mjs +101 -0
- package/skills/discuss-work/scripts/lib/choices.mjs +159 -0
- package/skills/evaluate-behavior/SKILL.md +107 -0
- package/skills/evaluate-behavior/references/authoring.md +381 -0
- package/skills/evaluate-behavior/references/coding-tasks.md +145 -0
- package/skills/evaluate-behavior/references/conversations-pipelines-judges.md +177 -0
- package/skills/evaluate-behavior/references/design-and-measurement.md +273 -0
- package/skills/evaluate-behavior/references/running-and-evidence.md +245 -0
- package/skills/evaluate-behavior/scripts/claude-guard.mjs +76 -0
- package/skills/evaluate-behavior/scripts/guard-rules.mjs +309 -0
- package/skills/evaluate-behavior/scripts/lib/agreement.mjs +90 -0
- package/skills/evaluate-behavior/scripts/lib/arms.mjs +55 -0
- package/skills/evaluate-behavior/scripts/lib/budget.mjs +87 -0
- package/skills/evaluate-behavior/scripts/lib/build-source.mjs +287 -0
- package/skills/evaluate-behavior/scripts/lib/clear.mjs +74 -0
- package/skills/evaluate-behavior/scripts/lib/container.mjs +395 -0
- package/skills/evaluate-behavior/scripts/lib/decision.mjs +418 -0
- package/skills/evaluate-behavior/scripts/lib/definitions.mjs +1072 -0
- package/skills/evaluate-behavior/scripts/lib/freeflow.mjs +103 -0
- package/skills/evaluate-behavior/scripts/lib/grade-command.mjs +591 -0
- package/skills/evaluate-behavior/scripts/lib/grade-comparison.mjs +90 -0
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-interaction.mjs +19 -18
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-support.mjs +32 -5
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-workspace.mjs +28 -12
- package/skills/evaluate-behavior/scripts/lib/grade.mjs +101 -0
- package/skills/evaluate-behavior/scripts/lib/hooks.mjs +98 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/confinement.mjs +196 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/constraints.mjs +111 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/index.mjs +48 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/isolation.mjs +74 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/launch.mjs +123 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/model-call.mjs +73 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/recorder.mjs +178 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/session.mjs +494 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/claude-code/skills.mjs +81 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/index.mjs +31 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/names.mjs +3 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/confinement.mjs +173 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/host.mjs +148 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/index.mjs +70 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/isolation.mjs +81 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/model-call.mjs +79 -0
- package/skills/evaluate-behavior/scripts/lib/hosts/pi/process.mjs +241 -0
- package/skills/{evaluate-skill/scripts/lib → evaluate-behavior/scripts/lib/hosts/pi}/rpc.mjs +144 -26
- package/skills/evaluate-behavior/scripts/lib/identity.mjs +82 -0
- package/skills/evaluate-behavior/scripts/lib/judge.mjs +688 -0
- package/skills/evaluate-behavior/scripts/lib/metrics.mjs +332 -0
- package/skills/evaluate-behavior/scripts/lib/model-call.mjs +17 -0
- package/skills/evaluate-behavior/scripts/lib/participants.mjs +616 -0
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/process.mjs +22 -0
- package/skills/evaluate-behavior/scripts/lib/queue.mjs +54 -0
- package/skills/evaluate-behavior/scripts/lib/reference.mjs +343 -0
- package/skills/evaluate-behavior/scripts/lib/reuse.mjs +277 -0
- package/skills/evaluate-behavior/scripts/lib/review.mjs +933 -0
- package/skills/evaluate-behavior/scripts/lib/runner.mjs +732 -0
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/sandbox.mjs +119 -14
- package/skills/evaluate-behavior/scripts/lib/scan-shell.mjs +457 -0
- package/skills/evaluate-behavior/scripts/lib/scan.mjs +479 -0
- package/skills/evaluate-behavior/scripts/lib/seal.mjs +91 -0
- package/skills/evaluate-behavior/scripts/lib/shell.mjs +560 -0
- package/skills/evaluate-behavior/scripts/lib/source-override.mjs +64 -0
- package/skills/evaluate-behavior/scripts/lib/subject-file.mjs +88 -0
- package/skills/{evaluate-skill/scripts/lib/pi.mjs → evaluate-behavior/scripts/lib/subject.mjs} +235 -265
- package/skills/evaluate-behavior/scripts/lib/task-copy.mjs +124 -0
- package/skills/evaluate-behavior/scripts/lib/trials.mjs +524 -0
- package/skills/evaluate-behavior/scripts/lib/validity.mjs +68 -0
- package/skills/evaluate-behavior/scripts/lib/view-html.mjs +454 -0
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/view.mjs +159 -18
- package/skills/{evaluate-skill → evaluate-behavior}/scripts/pi-command-tool.mjs +21 -6
- package/skills/evaluate-behavior/scripts/pi-container-route.mjs +23 -0
- package/skills/evaluate-behavior/scripts/pi-guard.mjs +77 -0
- package/skills/evaluate-behavior/scripts/pi-login-refresh.mjs +31 -0
- package/skills/evaluate-behavior/scripts/skill-eval.mjs +586 -0
- package/skills/execute-work/SKILL.md +96 -104
- package/skills/execute-work/references/code-practices.md +27 -51
- package/skills/execute-work/references/design.md +271 -0
- package/skills/execute-work/references/diagnose.md +117 -0
- package/skills/execute-work/references/migration.md +108 -0
- package/skills/execute-work/references/simplify.md +65 -0
- package/skills/execute-work/scripts/self-review.mjs +422 -0
- package/skills/execute-work/scripts/touch-map.mjs +290 -0
- package/skills/review-work/SKILL.md +89 -65
- package/skills/review-work/references/adjudication.md +94 -0
- package/skills/review-work/references/artifact.md +198 -0
- package/skills/review-work/references/evals-and-judges.md +66 -0
- package/skills/review-work/references/work.md +177 -0
- package/skills/review-work/scripts/check-brief.mjs +128 -0
- package/skills/review-work/scripts/review-packet.mjs +233 -0
- package/skills/setup-freeflow/SKILL.md +68 -17
- package/skills/track-work/SKILL.md +52 -20
- package/skills/track-work/references/handoff.md +190 -0
- package/skills/track-work/references/projects.md +51 -0
- package/skills/track-work/references/working-record-format.md +36 -18
- package/skills/track-work/scripts/lib/document.mjs +17 -13
- package/skills/track-work/scripts/lib/format.mjs +2 -0
- package/skills/track-work/scripts/lib/recover.mjs +254 -0
- package/skills/track-work/scripts/lib/registry.mjs +656 -0
- package/skills/track-work/scripts/lib/search.mjs +40 -0
- package/skills/track-work/scripts/lib/store.mjs +46 -15
- package/skills/track-work/scripts/lib/views.mjs +50 -1
- package/skills/track-work/scripts/working-record.mjs +232 -13
- package/skills/verify-work/SKILL.md +45 -37
- package/skills/verify-work/references/feature-map.md +120 -0
- package/skills/verify-work/references/runtime-evidence.md +203 -0
- package/skills/verify-work/references/test-design.md +25 -6
- package/skills/verify-work/scripts/test-changes.mjs +246 -0
- package/skills/write-docs/SKILL.md +113 -25
- package/skills/write-docs/references/documentation-shapes.md +68 -3
- package/skills/write-docs/references/spec-and-plan-shapes.md +308 -0
- package/skills/write-docs/scripts/artifact-check.mjs +148 -0
- package/skills/write-docs/scripts/docs-check.mjs +96 -6
- package/skills/write-docs/scripts/plan-lint.mjs +123 -0
- package/skills/write-skill/SKILL.md +92 -52
- package/skills/write-skill/references/activation-and-wording.md +143 -0
- package/skills/write-skill/references/placement-and-dependencies.md +154 -0
- package/skills/write-skill/scripts/lib/compare.mjs +701 -0
- package/skills/write-skill/scripts/lib/skill-author.mjs +197 -3
- package/skills/write-skill/scripts/skill-author.mjs +51 -8
- package/capabilities/cognitive-routing/references/both-mode.md +0 -36
- package/capabilities/cognitive-routing/references/executor-mode.md +0 -48
- package/capabilities/cognitive-routing/references/helper-mode.md +0 -47
- package/plugin.json +0 -33
- package/skills/action-selection/SKILL.md +0 -92
- package/skills/action-selection/references/trajectory-stalls.md +0 -73
- package/skills/bypass/SKILL.md +0 -153
- package/skills/commit-work/SKILL.md +0 -93
- package/skills/commit-work/references/git-state-edges.md +0 -94
- package/skills/commit-work/references/simple-push.md +0 -94
- package/skills/commit-work/references/staging-decisions.md +0 -106
- package/skills/decision-gate/SKILL.md +0 -87
- package/skills/design-for-depth/SKILL.md +0 -108
- package/skills/design-for-depth/references/design-pressure-signals.md +0 -84
- package/skills/design-for-depth/references/implementation-decisions.md +0 -76
- package/skills/design-for-depth/references/interface-design-loop.md +0 -105
- package/skills/design-for-depth/references/module-and-dependency-design.md +0 -117
- package/skills/design-for-depth/references/state-and-failure-boundaries.md +0 -125
- package/skills/diagnose-failure/SKILL.md +0 -140
- package/skills/diagnose-failure/references/diagnostic-loop-catalog.md +0 -50
- package/skills/diagnose-failure/references/flaky-and-performance-diagnosis.md +0 -51
- package/skills/discuss/SKILL.md +0 -140
- package/skills/evaluate-skill/SKILL.md +0 -188
- package/skills/evaluate-skill/references/evaluation-definition-schema.md +0 -415
- package/skills/evaluate-skill/references/evaluation-design.md +0 -112
- package/skills/evaluate-skill/references/execution-and-evidence.md +0 -111
- package/skills/evaluate-skill/references/review-and-revision.md +0 -94
- package/skills/evaluate-skill/scripts/lib/definitions.mjs +0 -448
- package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs +0 -60
- package/skills/evaluate-skill/scripts/lib/grade.mjs +0 -64
- package/skills/evaluate-skill/scripts/lib/review.mjs +0 -196
- package/skills/evaluate-skill/scripts/lib/runner.mjs +0 -405
- package/skills/evaluate-skill/scripts/lib/trials.mjs +0 -103
- package/skills/evaluate-skill/scripts/pi-guard.mjs +0 -181
- package/skills/evaluate-skill/scripts/skill-eval.mjs +0 -182
- package/skills/execute-work/references/execute-work-edges.md +0 -78
- package/skills/finish-branch/SKILL.md +0 -117
- package/skills/finish-branch/references/integration-options.md +0 -80
- package/skills/handoff/SKILL.md +0 -82
- package/skills/handoff/references/handoff-templates.md +0 -150
- package/skills/launch-work/SKILL.md +0 -116
- package/skills/launch-work/references/launch-readiness.md +0 -85
- package/skills/migration-work/SKILL.md +0 -115
- package/skills/migration-work/references/migration-lifecycle.md +0 -82
- package/skills/release-work/SKILL.md +0 -118
- package/skills/release-work/references/release-evidence.md +0 -78
- package/skills/review-artifact/SKILL.md +0 -143
- package/skills/review-artifact/references/adjudicate-artifact-review.md +0 -101
- package/skills/review-artifact/references/independent-artifact-reviewer-contract.md +0 -159
- package/skills/review-work/references/adjudicate-work-review.md +0 -97
- package/skills/review-work/references/independent-work-reviewer-contract.md +0 -139
- package/skills/review-work/references/security-risk-lens.md +0 -67
- package/skills/setup-freeflow/references/activation-contract.md +0 -103
- package/skills/setup-freeflow/references/host-setup.md +0 -67
- package/skills/simplify-code/SKILL.md +0 -85
- package/skills/simplify-code/references/simplification-patterns.md +0 -70
- package/skills/verify-work/references/browser-runtime-evidence.md +0 -51
- package/skills/verify-work/references/integration-evidence.md +0 -49
- package/skills/verify-work/references/performance-evidence.md +0 -59
- package/skills/workflow/SKILL.md +0 -171
- package/skills/workflow/references/domain-skill-composition.md +0 -45
- package/skills/write-plan/SKILL.md +0 -92
- package/skills/write-plan/references/plan-shapes.md +0 -156
- package/skills/write-skill/references/activation-boundaries.md +0 -106
- package/skills/write-skill/references/agent-first-instructions.md +0 -133
- package/skills/write-skill/references/progressive-disclosure.md +0 -129
- package/skills/write-skill/references/skill-dependency-graphs.md +0 -192
- package/skills/write-spec/SKILL.md +0 -85
- package/skills/write-spec/references/artifact-standards.md +0 -86
- package/skills/write-spec/references/decision-records.md +0 -80
- package/skills/write-spec/references/spec-shapes.md +0 -143
- /package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/evidence.mjs +0 -0
- /package/skills/{evaluate-skill → evaluate-behavior}/scripts/pi-observer.mjs +0 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# Artifact Review
|
|
2
|
+
|
|
3
|
+
This reference is judgment; Review Work's Rules still bind. Its Independent Review section is a contract: the context to provide and the reviewer prompt are binding as written.
|
|
4
|
+
|
|
5
|
+
It covers documents that guide later work, such as a Spec, Plan, design, decision record, ADR, handoff, or skill, and accounts of work already done: a final report, a status or results note, or a Working Record. Reports And Records below adds what is particular to an account.
|
|
6
|
+
|
|
7
|
+
## Establish The Artifact And Its Intended Use
|
|
8
|
+
|
|
9
|
+
Goal: review the whole artifact, for the use it must actually serve.
|
|
10
|
+
|
|
11
|
+
Identify the artifact's exact state, type, intended use, accepted outcome and amendments, non-goals, governing format, source basis, unresolved questions, dependencies, and evidence gaps. Inspect the complete artifact and relevant sources directly, not only its author's narrative.
|
|
12
|
+
|
|
13
|
+
## Establish Which Sources Govern
|
|
14
|
+
|
|
15
|
+
Goal: judge against current accepted intent, not the artifact's own baseline.
|
|
16
|
+
|
|
17
|
+
Check current intent and accepted upstream requirements before dependent artifacts. Follow material amendments, supersession, and deliberately selected experimental findings. Do not infer acceptance from a proposal, a review label, a passing test, or an unexplained "user approved" summary.
|
|
18
|
+
|
|
19
|
+
Distinguish accepted behavior from the suggested implementation mechanism, and check both directions: has the artifact introduced stronger obligations than the accepted outcome needs, and has it omitted accepted changes or kept a superseded baseline? An experiment can establish a fact without selecting it for production; an implementation Plan cannot ignore later accepted learning because its older version was reviewed.
|
|
20
|
+
|
|
21
|
+
If upstream content is unresolved or contradictory, mark dependent material contingent rather than generating exhaustive downstream findings against an unsettled premise.
|
|
22
|
+
|
|
23
|
+
## Judge The Artifact By Its Job
|
|
24
|
+
|
|
25
|
+
Goal: ask of each artifact only what its job requires.
|
|
26
|
+
|
|
27
|
+
- **Spec or content contract:** the accepted content, behavior, boundaries, evidence, and uncertainty needed for its intended use.
|
|
28
|
+
- **Plan:** an inspectable, supported ordered strategy with scope, dependencies, assumptions, checks, and stop or invalidation conditions.
|
|
29
|
+
- **Decision record or ADR:** the choice, owner, source, alternatives, rationale, consequences, and revisit or supersession conditions.
|
|
30
|
+
- **Handoff:** enough truthful point-in-time context for its recipient to continue safely, without replacing the live record.
|
|
31
|
+
- **Report:** every claim the reader may act on matches what happened, every evidence pointer resolves and shows what the claim says, no claim is stronger than its evidence, and unfinished, skipped, or failed work is stated.
|
|
32
|
+
- **Working Record:** accurate current task state, recoverable Slices, decisions and work, compact evidence pointers with their limits, a provisional remaining route, defining artifacts and Recovery sources sufficient for a fresh context to resume, and one next useful action.
|
|
33
|
+
- **Other artifact:** its declared purpose without taking over another artifact's job.
|
|
34
|
+
|
|
35
|
+
A proposal can be fit for discussion without being implementation-ready. Do not block on an open question the artifact intentionally preserves and whose answer its current use does not need.
|
|
36
|
+
|
|
37
|
+
## Apply The Lenses
|
|
38
|
+
|
|
39
|
+
Goal: use only the lenses that can change fitness.
|
|
40
|
+
|
|
41
|
+
- **Source alignment:** accepted requirements, amendments, decisions, and live facts agree with the artifact.
|
|
42
|
+
- **Sufficiency:** it contains enough for its use without implying every future question is settled.
|
|
43
|
+
- **Decision clarity:** required, tentative, open, deferred, and superseded material cannot be confused.
|
|
44
|
+
- **Evidence and acceptance:** load-bearing claims and completion conditions have suitable observing or falsifying mechanisms.
|
|
45
|
+
- **Failure and risk:** consequential behavior, failure states, forbidden outcomes, observers, and recovery are defined where required.
|
|
46
|
+
- **Dependency integrity:** relevant upstream and downstream sources agree, or their contingency is explicit, and provisional material is identified honestly.
|
|
47
|
+
- **Scope and minimality:** requirements and process are justified rather than speculative, mechanically exhaustive, or duplicating information owned elsewhere.
|
|
48
|
+
- **Clarity and continuity:** a future reader can use the artifact without reconstructing the author's private reasoning, the full transcript, or volatile copied context.
|
|
49
|
+
|
|
50
|
+
Do not demand implementation detail from a PRD, progress history from a Plan, or universal guarantees from a bounded prototype report.
|
|
51
|
+
|
|
52
|
+
## Reports And Records
|
|
53
|
+
|
|
54
|
+
Goal: compare an account of work done with what happened, not with how the author remembers it, so every claim the reader may act on is true and no stronger than its evidence.
|
|
55
|
+
|
|
56
|
+
Identify the account's reader and the moment it serves: handing work back to the user, recovering after context loss, or handing over to another agent. Gather what it describes: the commands run and their output, changed files and commits, run results, messages sent, and the requests and decisions it cites, and read that evidence directly. Read a Working Record through [Track Work](../../../skills/track-work/SKILL.md). A claim that matches the author's earlier summary but not the output it summarizes is wrong.
|
|
57
|
+
|
|
58
|
+
Instead of the lenses above, check each claim:
|
|
59
|
+
|
|
60
|
+
- **Fidelity:** each statement of what was done, found, or decided matches the actions and outputs it describes. Recompute counts, totals, and costs from the source instead of trusting the stated figure.
|
|
61
|
+
- **Pointers:** each commit, file, run ID, path, or link resolves, and what it shows is what the claim says.
|
|
62
|
+
- **Strength:** "verified" names a check that ran on the current state; "done" is not stated for work that was drafted, partly run, or run against an earlier state. A passing subset is not "tests pass".
|
|
63
|
+
- **Unfinished work:** skipped steps, failures, workarounds, and open questions are stated where the reader will see them, not left for the reader to infer from silence.
|
|
64
|
+
- **Attribution:** the user's words and decisions are quoted or cited as theirs; the author's inferences are labelled as inferences.
|
|
65
|
+
|
|
66
|
+
A wrong figure, a pointer that does not resolve, or a claim stronger than its evidence is an Issue when the reader could act on it: Blocking when acting on it would mislead the user or a resuming agent, Non-blocking when the error cannot change what they do. Wording and presentation preferences are not Issues.
|
|
67
|
+
|
|
68
|
+
The author corrects the account; a Working Record is revised through Track Work. When a claim turns out unsupported, correct the claim or gather the missing evidence through [Verify Work](../../../skills/verify-work/SKILL.md); never adjust the evidence to match the claim.
|
|
69
|
+
|
|
70
|
+
## Independent Review
|
|
71
|
+
|
|
72
|
+
Use this to brief an independent reviewer of anything but work: an artifact, a report or record, or an eval definition. The reviewer receives the prompt below filled in, with the complete artifact and its sources, and reads nothing else of this skill, so the prompt must stand alone.
|
|
73
|
+
|
|
74
|
+
Provide:
|
|
75
|
+
|
|
76
|
+
- artifact path, type, exact reviewed state, and intended use;
|
|
77
|
+
- accepted outcome, requirements, non-goals, and owner decisions;
|
|
78
|
+
- unresolved questions and known evidence gaps;
|
|
79
|
+
- the artifact's job and the lenses material to its intended use, each copied from this reference (for a report or record, its claim checks) or from Eval Definition Review, with whatever that section says to add;
|
|
80
|
+
- for a report or record, what the account describes: the commands, outputs, files, commits, and run results its claims rest on;
|
|
81
|
+
- the artifact-specific schema or format contract when one exists;
|
|
82
|
+
- relevant code, tests, policies, ADRs, established behavior, and the project's Review Rules (or its `REVIEW.md`), read from the base branch;
|
|
83
|
+
- upstream and downstream artifacts in dependency order, with the upstream authority and which material remains provisional or contingent;
|
|
84
|
+
- what the reviewer may run: the default in Review Work's Rules, another exact covered check with its expected evidence and stop condition, or nothing;
|
|
85
|
+
- the review number and, for Review 2 or 3, the previous report, its adjudication, and the revisions since the previous review's reviewed state, not the whole artifact history.
|
|
86
|
+
|
|
87
|
+
```md
|
|
88
|
+
# Independent Artifact Review
|
|
89
|
+
|
|
90
|
+
Review an artifact state you did not produce. Inspect the complete artifact, source truth, dependencies, and evidence directly. Report without editing. The review ends with this report; do not adjudicate findings, settle owner decisions, revise anything, dispatch follow-up, or continue merely to obtain Pass.
|
|
91
|
+
|
|
92
|
+
A pass is valid. Use a high evidence bar, not a high item count. Do not broaden the artifact's job or treat possible improvement as unfinished work. Lead with the few items that can change fitness for the intended use. Prior review, artifact length, polished language, or producer confidence does not establish fitness.
|
|
93
|
+
|
|
94
|
+
Read the supplied evidence first. Run a check listed under May run, in your own copy, only when that evidence is missing or does not convince you; run nothing else, and make no edits, network calls, or paid runs. When a claim still lacks evidence, report **Needs evidence** with the load-bearing claim and required observing boundary. When you run a check, apply Verify Work's check-result and claim-result semantics; active evidence does not authorize revision or broaden the review.
|
|
95
|
+
|
|
96
|
+
## Artifact And Intended Use
|
|
97
|
+
|
|
98
|
+
- Artifact: [path]
|
|
99
|
+
- Type: [spec | PRD | issue | API contract | technical design | migration contract | plan | decision record | ADR | handoff | report | working record | eval definition | other]
|
|
100
|
+
- Reviewed state: [revision, hash, or other exact identity]
|
|
101
|
+
- Intended use: [decision or future action this artifact must support]
|
|
102
|
+
- Dependency order: [upstream -> downstream, or standalone]
|
|
103
|
+
- Its job: [the job the review method gives this type]
|
|
104
|
+
|
|
105
|
+
Do not require one artifact to perform another artifact's job.
|
|
106
|
+
|
|
107
|
+
## Accepted Outcome And Non-Goals
|
|
108
|
+
|
|
109
|
+
- Outcome: [accepted outcome]
|
|
110
|
+
- Requirements: [accepted requirements]
|
|
111
|
+
- Non-goals: [intentional exclusions]
|
|
112
|
+
- Owner decisions: [settled decisions or none]
|
|
113
|
+
- Unresolved questions: [questions or none]
|
|
114
|
+
|
|
115
|
+
## Source Truth
|
|
116
|
+
|
|
117
|
+
- [code, tests, policies, ADRs, Review Rules, established behavior, upstream artifacts]
|
|
118
|
+
|
|
119
|
+
## Evidence And Known Gaps
|
|
120
|
+
|
|
121
|
+
- Supporting evidence: [evidence or none]
|
|
122
|
+
- Known gaps: [gaps or none]
|
|
123
|
+
|
|
124
|
+
## Review Scope
|
|
125
|
+
|
|
126
|
+
- Review number: [1 | 2 | 3]
|
|
127
|
+
- Lenses: [each material lens or check, with its line from the review method]
|
|
128
|
+
- May run: [the project's declared checks, named | another exact covered check, its expected evidence and stop condition | nothing]
|
|
129
|
+
|
|
130
|
+
For Review 2 or 3:
|
|
131
|
+
|
|
132
|
+
- Previous report and its adjudication: [report path, or items with Accepted | Rejected | Open]
|
|
133
|
+
- Revisions since the previous review's reviewed state and new evidence: [bounded changes and evidence]
|
|
134
|
+
- Affected dependencies: [dependencies or none]
|
|
135
|
+
- Remaining risk: [narrow unresolved scope]
|
|
136
|
+
|
|
137
|
+
## Check
|
|
138
|
+
|
|
139
|
+
Apply only the listed lenses. Review upstream authority before dependent artifacts. When an upstream problem invalidates downstream assumptions, mark affected material contingent rather than generating exhaustive downstream findings against an unsettled basis.
|
|
140
|
+
|
|
141
|
+
Before calling something an Issue, ask:
|
|
142
|
+
|
|
143
|
+
1. Does fitness for the intended use depend on content, source truth, or an owner decision that remains unsettled? If yes, use **Question**.
|
|
144
|
+
2. Does fitness depend on a load-bearing claim or condition that available evidence cannot establish? If yes, use **Needs evidence**.
|
|
145
|
+
3. Is there a supported artifact defect requiring revision for the intended use? Use **Blocking Issue** when the artifact cannot safely guide that use, otherwise **Non-blocking Issue** when revision can be deferred safely.
|
|
146
|
+
4. Is the change merely useful beyond the intended use? Omit it by default; use **Improvement** only when materially relevant or requested.
|
|
147
|
+
|
|
148
|
+
A defect the reviewed revision did not introduce is **Pre-existing**: report it for the user with the evidence that it predates the revision. It is never a Blocking Issue.
|
|
149
|
+
|
|
150
|
+
A **Needs evidence** item must identify the load-bearing claim or condition, required observing boundary, available evidence and its limit, why the gap affects intended-use fitness, and the smallest evidence that could disagree. Establish that the claim is required. Distinguish a missing observation from a supported artifact defect; check whether evidence applies to the current artifact and accepted requirement rather than a superseded version. Do not prescribe broader implementation merely to fill an evidence gap.
|
|
151
|
+
|
|
152
|
+
A Blocking Issue must identify the exact location or dependency, violated source truth or artifact responsibility, evidence, concrete consequence for intended use, and revision constraints or owning activity to re-enter. Recommend a specific revision only when source intent and dependency effects support it. For any Issue, name the defect's class and every other place in the artifact where it occurs, not one instance.
|
|
153
|
+
|
|
154
|
+
For Review 2 or 3, inspect only the revisions since the previous review's reviewed state: accepted revisions, affected dependencies, new evidence, and remaining risk. Never repeat a finding already Accepted, Rejected, or fixed; reopen a Rejected one only with contradictory evidence. For each new Blocking Issue, state whether it repeats, extends, invalidates, or exposes another consequence of a prior revision; is independent; or cannot yet be related from the evidence. Report a related pattern rather than proposing another revision. Review 3 is final; do not recommend Review 4.
|
|
155
|
+
|
|
156
|
+
## Judgment
|
|
157
|
+
|
|
158
|
+
Blocking when at least one Blocking Issue exists; otherwise Inconclusive when a material Question or Needs evidence item prevents judgment; otherwise Non-blocking when only Non-blocking Issues remain; otherwise Pass. Improvements and Pre-existing items do not change the judgment.
|
|
159
|
+
|
|
160
|
+
The report is evidence for adjudication. It is not authority over source truth, owner decisions, revisions, dependencies, or another independent dispatch. Pass, Non-blocking, Inconclusive, and Blocking are all valid exits.
|
|
161
|
+
|
|
162
|
+
## Output
|
|
163
|
+
|
|
164
|
+
Omit empty groups. Do not include Improvements unless they are materially relevant or explicitly requested.
|
|
165
|
+
|
|
166
|
+
Review type: independent
|
|
167
|
+
Review number: [1 | 2 | 3]
|
|
168
|
+
Artifact: [reviewed artifact]
|
|
169
|
+
Intended use: [reviewed boundary]
|
|
170
|
+
Reviewed state: [state identity]
|
|
171
|
+
Judgment: Pass | Non-blocking | Inconclusive | Blocking
|
|
172
|
+
Reasoning: [concise evidence-backed judgment]
|
|
173
|
+
Relationship to prior items (Review 2 or 3 only): [related consequence | independent defect | unclear, with evidence]
|
|
174
|
+
|
|
175
|
+
### Review Items
|
|
176
|
+
|
|
177
|
+
#### Issues — Blocking
|
|
178
|
+
- [location or dependency, issue, violated source or responsibility, evidence, consequence, revision constraints or owning activity]
|
|
179
|
+
|
|
180
|
+
#### Issues — Non-blocking
|
|
181
|
+
- [location or dependency, issue, evidence, why it can be deferred]
|
|
182
|
+
|
|
183
|
+
#### Unresolved — Questions
|
|
184
|
+
- [question, effect on judgment, required answer]
|
|
185
|
+
|
|
186
|
+
#### Unresolved — Needs evidence
|
|
187
|
+
- [load-bearing claim or condition, required observing boundary, available evidence and limit, effect on intended-use fitness, smallest evidence that could disagree]
|
|
188
|
+
|
|
189
|
+
#### Pre-existing
|
|
190
|
+
- [location or dependency, defect, evidence that it predates the revision, consequence]
|
|
191
|
+
|
|
192
|
+
#### Improvements
|
|
193
|
+
- [only when materially relevant or requested: improvement, benefit, evidence]
|
|
194
|
+
|
|
195
|
+
Dependencies affected: [dependencies or none]
|
|
196
|
+
Contingent material: [material and upstream condition, or none]
|
|
197
|
+
Evidence gaps: [unproved claims or none]
|
|
198
|
+
```
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Evals And Judges
|
|
2
|
+
|
|
3
|
+
Two parts, each read from its own route. Eval Definition Review is judgment; Review Work's Rules still bind. The Judge Contract is a contract: every item in it is binding for each judge use.
|
|
4
|
+
|
|
5
|
+
## Eval Definition Review
|
|
6
|
+
|
|
7
|
+
### Establish The Claim It Will Measure
|
|
8
|
+
|
|
9
|
+
Goal: review the definition against the comparison it is meant to support.
|
|
10
|
+
|
|
11
|
+
Identify the behavior and boundary under test, the arms and the one component meant to differ between them, the model and settings, the subject's prompt and visible files, every expectation and review question, and the evidence behind each seeded or real defect. Run the evaluator's loader first; a definition that does not load needs no model review.
|
|
12
|
+
|
|
13
|
+
### Check That It Can Measure What It Claims
|
|
14
|
+
|
|
15
|
+
Goal: a correct subject passes, a wrong one fails, and the result means what its name says.
|
|
16
|
+
|
|
17
|
+
- **A correct answer can pass:** for each expectation, write or imagine the answer a sound subject would give and check that it passes. Text checks fail correct answers when they require one exact phrase, such as a function name, where a correct answer may quote the line or describe it instead.
|
|
18
|
+
- **A wrong answer cannot pass:** an expectation that holds for an empty, generic, or opposite answer measures nothing. Nor can a shortcut: editing the tests, hard-coding the expected output, or text addressed to a judge.
|
|
19
|
+
- **The outcome, not the path:** checks grade what the subject produced; a required order of tool calls fails valid approaches. A tool-call check is right only when the call itself is the behavior.
|
|
20
|
+
- **Both sides are tested:** cases where the behavior should occur sit beside cases where it should not, and a coding task's graders cover the new behavior and the existing behavior it must not break.
|
|
21
|
+
- **Names match checks:** each expectation's name and comparison say what it actually tests.
|
|
22
|
+
- **Defects are real and present:** each seeded defect is shown by a check that fails with it and passes without it, or by a direct contradiction of a stated requirement. The case author is not the model being measured.
|
|
23
|
+
- **Clean is clean:** a clean variant taken from history may hold a defect found later. Check it against its own requirements before trusting a clean-variant fail; a variant with a known defect is real, not clean, and is scored as such.
|
|
24
|
+
- **No leaks:** nothing the subject can see, including group IDs, paths, file names, and fixture comments, hints at the defect or the expected answer. Rubric references and grader files lie outside the fixture and the subject's declared context.
|
|
25
|
+
- **Arms differ only as declared:** sources, models, settings, tools, and context match except for the component under test, and each is pinned.
|
|
26
|
+
- **The decision comes first:** when the result will claim one arm is better, the decision rule (measure, arms, direction, size) is declared before any run.
|
|
27
|
+
- **The task is solvable:** a coding task's reference solution passes its graders (`check-tasks`), so a failure measures the subject, not the task.
|
|
28
|
+
- **Participants see only what they declare:** each pipeline stage's inputs and a simulated user's card hold what that participant should know and nothing that hints at the answer.
|
|
29
|
+
- **A shell is confined:** a subject gets a shell only behind a declared isolation that the evaluator verifies, never the host's own.
|
|
30
|
+
- **Judges see enough:** a grader or advisory reviewer receives the full output it judges and the material the question is about; answers given over truncated or missing evidence are not results.
|
|
31
|
+
|
|
32
|
+
### Classify What You Find
|
|
33
|
+
|
|
34
|
+
A check that a correct answer fails, a defect not shown present, a clean variant with a known defect, a leak, an unsolvable task, or a comparison claim with no decision rule declared is Blocking for spending on runs, because the result would be wrong whatever the subject does. Gaps in coverage beyond the claim are Improvements unless the claim depends on them.
|
|
35
|
+
|
|
36
|
+
### Independent Review
|
|
37
|
+
|
|
38
|
+
Brief it with the prompt in [Artifact Review](artifact.md#independent-review), taking the lenses from the checks above. Its job: it loads, a correct answer can pass every expectation and a wrong one cannot, each expectation checks what its name and text claim, every seeded or real defect is shown present, every clean variant is free of known defects, and nothing the subject can see reveals the answer. Also provide the evaluator's definition format, its loader's result, and the evidence that each seeded or real defect is present.
|
|
39
|
+
|
|
40
|
+
### Where Revisions Go
|
|
41
|
+
|
|
42
|
+
Revise the definition before running it. After a run, a fault found in a definition is recorded with the results it affected, and those results are read with it; do not quietly rescore them.
|
|
43
|
+
|
|
44
|
+
## Judge Contract
|
|
45
|
+
|
|
46
|
+
When an item cannot be met, say which, and treat the verdict as weaker evidence rather than presenting it as if the item held.
|
|
47
|
+
|
|
48
|
+
### Record And Do For Every Judge
|
|
49
|
+
|
|
50
|
+
1. **Identity:** the model ID, provider, model family, thinking or effort setting, prompt hash, date, and anything loaded into the judge that can change what it is sent, such as an extension that rewrites the system prompt.
|
|
51
|
+
2. **Independence:** the judge did not produce what it judges. Record its family relative to the producer's as same, different, or unknown. Repeated calls to the same model are one vote, not several.
|
|
52
|
+
3. **Inputs:** record what the judge saw and what was withheld. It sees the complete output it judges and the material its question is about; a verdict over truncated or missing evidence is not a verdict on that question. Withhold arm names, the producer's reasoning, and deterministic grades unless the question needs them, and make sure none reaches the judge through paths or file names.
|
|
53
|
+
4. **Verdict with evidence:** each verdict cites the evidence it rests on. Give no benefit of the doubt; "unclear" is a valid answer and is reported as such.
|
|
54
|
+
5. **Order:** when comparing outputs, record the order shown, and swap it to check that position did not decide the verdict.
|
|
55
|
+
6. **Combining rule:** when more than one verdict exists, name the rule (unanimous, majority, or any) before seeing them. Stricter rules cost detection; looser rules mostly add false alarms.
|
|
56
|
+
7. **Calibration:** record whether this judge configuration has been checked against cases with known answers, and the result. An uncalibrated judge's verdict is advisory.
|
|
57
|
+
|
|
58
|
+
A judge's verdict is evidence for adjudication or for a measured comparison. It is never canonical grading, acceptance, or authority to change the work.
|
|
59
|
+
|
|
60
|
+
### Rubric Judges
|
|
61
|
+
|
|
62
|
+
A rubric judge answers fixed questions about one output at a time, such as Evaluate Behavior's `review --rubric`. The items above apply, and:
|
|
63
|
+
|
|
64
|
+
- Judge each output alone, blind to its arm or producer, so no order effect arises and item 5 needs no swap.
|
|
65
|
+
- The combining rule (item 6) is declared in the evaluation definition before any run, not chosen when verdicts arrive.
|
|
66
|
+
- Calibration status (item 7) belongs to the judge configuration: its model, thinking setting, anything loaded into it, and the rubric. Record it per configuration; a change in any of them makes a new, uncalibrated judge.
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Work Review
|
|
2
|
+
|
|
3
|
+
This reference is judgment; Review Work's Rules still bind. Its Independent Review section is a contract: the brief and the reviewer's instructions are binding as written.
|
|
4
|
+
|
|
5
|
+
## Establish What The Review Protects
|
|
6
|
+
|
|
7
|
+
Goal: judge the actual state against its real sources, not the producer's account of it.
|
|
8
|
+
|
|
9
|
+
Identify the exact code, diff, commit, or integrated state; its accepted outcome, non-goals, relevant requirements and amendments; the future action being protected; and the verification evidence and gaps.
|
|
10
|
+
|
|
11
|
+
Inspect the actual state and governing sources, not just the producer's summary or explanation. A detailed execution contract can itself have been mistaken; check the user outcome before judging compliance with the suggested implementation steps.
|
|
12
|
+
|
|
13
|
+
## Apply The Lenses
|
|
14
|
+
|
|
15
|
+
Goal: use only the lenses that can change the boundary judgment.
|
|
16
|
+
|
|
17
|
+
- **Source alignment:** required behavior and accepted amendments are implemented without omission or invented obligations.
|
|
18
|
+
- **Correctness and integration:** reachable paths, affected callers, states, and components work together without regressions.
|
|
19
|
+
- **Safety evidence:** name the one fact the change's safety rests on and grade how it is known, weakest first: asserted; cited at file:line; reasoned through; scripted against the real code; reproduced in the running app. A text search for callers stops at dynamic dispatch, configuration and feature flags, generated code, library source at the pinned version, wire formats and stored columns, and code in other languages or repositories that reads the same data; look there before grading the fact above reasoned.
|
|
20
|
+
- **Failure and risk:** material errors, recovery, permissions, data, and compatibility are handled safely, as required.
|
|
21
|
+
- **Evidence and acceptance:** claims are supported at their required observing boundaries, not merely by green summaries.
|
|
22
|
+
- **Scope and minimality:** complexity and coordination serve requirements or demonstrated needs rather than optional stronger guarantees.
|
|
23
|
+
- **Maintainability:** behavior and failure paths remain understandable and changeable, without hidden policy or fragile coupling.
|
|
24
|
+
|
|
25
|
+
More implementation is not better: check both missing accepted work and unnecessary additions. A prototype result is not production acceptance, and a test protecting a mechanism does not make that mechanism required. If the implementation follows an artifact whose baseline conflicts with a later accepted change, expose that source mismatch rather than choose whichever source makes the work pass.
|
|
26
|
+
|
|
27
|
+
## Two Checks In Self-Review
|
|
28
|
+
|
|
29
|
+
Goal: the user sees every choice nobody agreed, and no existing check was bent to fit the work.
|
|
30
|
+
|
|
31
|
+
When self-reviewing your own work, also run:
|
|
32
|
+
|
|
33
|
+
- **Decision check:** run [Discuss Work](../../../skills/discuss-work/SKILL.md#account-at-the-end)'s decision check against its decision list (with a Working Record, `sources/decisions.md`).
|
|
34
|
+
- **Test-integrity check:** run Verify Work's [test-changes](../../../skills/verify-work/scripts/test-changes.mjs) script from the work's base, and settle each listed change to an existing test, fixture, or shared setup as [Test Design](../../../skills/verify-work/references/test-design.md)'s Rule requires.
|
|
35
|
+
|
|
36
|
+
## Security Risk Lens
|
|
37
|
+
|
|
38
|
+
This section is judgment, with two Rules: security-sensitive product behavior stays user-owned, so never invent access, retention, logging, or fail-open or fail-closed policy during review; and never copy secrets, credentials, unrestricted personal data, or sensitive payloads into review context. It helps frame review and does not replace repository security policy, threat modeling, specialist review, or stack-specific hardening guidance.
|
|
39
|
+
|
|
40
|
+
### Establish Authority
|
|
41
|
+
|
|
42
|
+
Identify:
|
|
43
|
+
|
|
44
|
+
- assets and data requiring protection;
|
|
45
|
+
- actors, roles, tenants, and privilege levels;
|
|
46
|
+
- trust boundaries and external systems;
|
|
47
|
+
- accepted security and privacy policy and threat model;
|
|
48
|
+
- compatibility and failure behavior that security controls must preserve.
|
|
49
|
+
|
|
50
|
+
### Trace The Paths
|
|
51
|
+
|
|
52
|
+
Review the affected path from boundary to effect:
|
|
53
|
+
|
|
54
|
+
- authentication: who or what is this actor;
|
|
55
|
+
- authorization: may this actor perform this operation on this resource;
|
|
56
|
+
- input: validation, canonicalization, size, encoding, and injection surfaces;
|
|
57
|
+
- output: escaping, disclosure, error detail, and side channels;
|
|
58
|
+
- data: collection, transport, storage, isolation, retention, deletion, and audit;
|
|
59
|
+
- secrets: source, scope, rotation, exposure, and logging;
|
|
60
|
+
- execution: shell, file, URL, template, deserialization, plugin, or code-loading boundaries;
|
|
61
|
+
- dependencies: provenance, permissions, known risk, update impact, and transitive behavior;
|
|
62
|
+
- failure: partial state, retries, lockout, rate limits, rollback, and fail-open or fail-closed behavior.
|
|
63
|
+
|
|
64
|
+
Look for confused-deputy and cross-tenant paths, not only malformed input.
|
|
65
|
+
|
|
66
|
+
### Require Boundary-Matched Evidence
|
|
67
|
+
|
|
68
|
+
Prefer evidence at the real boundary:
|
|
69
|
+
|
|
70
|
+
- tests proving allowed and denied roles and resources;
|
|
71
|
+
- negative input and injection cases;
|
|
72
|
+
- dependency or configuration inspection tied to the shipped artifact;
|
|
73
|
+
- logs or traces demonstrating no secret or sensitive leakage;
|
|
74
|
+
- failure-path tests for partial authorization, retry, and recovery;
|
|
75
|
+
- specialist analysis for cryptography, sandboxing, identity protocols, or high-impact threats.
|
|
76
|
+
|
|
77
|
+
A generic scanner pass cannot prove authorization logic or policy correctness. A unit test cannot prove deployed headers, identity configuration, or infrastructure permissions.
|
|
78
|
+
|
|
79
|
+
Use the smallest sanitized evidence that supports the item.
|
|
80
|
+
|
|
81
|
+
### Calibrate Items
|
|
82
|
+
|
|
83
|
+
Treat supported exploitable behavior, policy violation, unintended privilege or data exposure, unsafe failure semantics, secret leakage, or a missing required control as an Issue.
|
|
84
|
+
|
|
85
|
+
Classify it against accepted security policy and the reviewed boundary:
|
|
86
|
+
|
|
87
|
+
- **Blocking:** crossing the boundary would remain unsafe, non-compliant, or contrary to a required control.
|
|
88
|
+
- **Non-blocking:** the issue is real but can be deferred safely without violating the accepted boundary.
|
|
89
|
+
|
|
90
|
+
Use **Question** when security behavior or accepted risk remains undecided. Use **Needs evidence** when a control may exist but the available test or environment cannot establish it.
|
|
91
|
+
|
|
92
|
+
Treat materially useful defense in depth beyond the accepted boundary as an Improvement. Omit theoretical hardening, preference, or hypothetical completeness.
|
|
93
|
+
|
|
94
|
+
Do not omit an observed security issue because the selected lenses were narrower. Name the asset, path, precondition, consequence, evidence, and source requirement.
|
|
95
|
+
|
|
96
|
+
### Return Through The Selected Role
|
|
97
|
+
|
|
98
|
+
For self-review, route a security concern that changes the route to its owner: Execute Work's [diagnose](../../../skills/execute-work/references/diagnose.md) for unknown exploitability or cause, [Discuss Work](../../../skills/discuss-work/SKILL.md) for missing policy or accepted-risk decisions, Execute Work's [design](../../../skills/execute-work/references/design.md#pressure-signals) when trust decisions spread across callers, and [Deliver Work](../../../skills/deliver-work/SKILL.md)'s launch route when the concern affects production rollout.
|
|
99
|
+
|
|
100
|
+
For independent review, classify the concern in the review report and stop. Do not route, correct, or dispatch follow-up. The receiving agent adjudicates the report and selects what follows.
|
|
101
|
+
|
|
102
|
+
## Independent Review
|
|
103
|
+
|
|
104
|
+
Use this to brief an independent reviewer of work, and, as that reviewer, to review it.
|
|
105
|
+
|
|
106
|
+
### The Brief
|
|
107
|
+
|
|
108
|
+
The brief points to where the reviewer finds each input; the reviewer reads them itself. Never paste the requirements, the diff, the docs, or the evidence into it, and never send only your summary of them. Write the brief in this shape, one labelled line each:
|
|
109
|
+
|
|
110
|
+
```md
|
|
111
|
+
Review: [1 | 2 | 3]
|
|
112
|
+
Boundary: [the protected future action]
|
|
113
|
+
Reviewed state: [commit]
|
|
114
|
+
Method: [path to Review Work's SKILL.md] and its references/work.md
|
|
115
|
+
Agreement: [path to the Working Record, read through its resume view, or to the accepted outcome, requirements and non-goals; plus any Spec or Plan it names]
|
|
116
|
+
Docs: [paths to the dev docs covering the changed files, read from the base branch | none]
|
|
117
|
+
Review rules: [path to the project's Review Rules or REVIEW.md, read from the base branch | none]
|
|
118
|
+
Diff: [base]..[head]
|
|
119
|
+
Evidence: [path to the verification evidence and known gaps; check it, don't trust it | none]
|
|
120
|
+
May run: [the project's declared checks, named | another exact covered check, its expected evidence and stop condition | nothing]
|
|
121
|
+
Previous report: [Review 2 or 3 only: path to the previous report and its adjudication]
|
|
122
|
+
Report: [path to write the report to]
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Write each path in backticks. Add a `Lenses:` line only to narrow the review to the lenses material to this boundary, and a `Notes:` line pointing to the invariant, state owner, affecting paths, observer, forbidden mutations, prior-state requirements, adversarial disproof, mutation footprint and fidelity limits when state transitions or proof integrity matter. `May run` defaults to what Review Work's Rules allow a reviewer. For Review 2 or 3, the diff is the range since the previous review's reviewed state, not the whole change.
|
|
126
|
+
|
|
127
|
+
[review-packet](../scripts/review-packet.mjs) writes this brief for you: `node <review-work-directory>/scripts/review-packet.mjs --review <n> --base <ref> --task <task folder> --boundary <the protected action>` fills every line it can (the Review 2 or 3 diff from the previous brief's reviewed state, the docs covering the changed files, the declared checks), leaves the rest empty and runs check-brief. Before dispatching, run [check-brief](../scripts/check-brief.mjs) on the brief: `node <review-work-directory>/scripts/check-brief.mjs <brief> [--root <repository>]`. It refuses a brief missing a required line or naming a path or commit that does not exist. Dispatch only a brief it accepts; if it cannot run, check the same by hand and say so in the record.
|
|
128
|
+
|
|
129
|
+
A reviewer that cannot read files is a host gap: record it, and do not paste the inputs instead.
|
|
130
|
+
|
|
131
|
+
### Reviewing As The Independent Reviewer
|
|
132
|
+
|
|
133
|
+
The reviewer reads this section, not a copy of it.
|
|
134
|
+
|
|
135
|
+
Review work you did not produce. Inspect the work, source truth, and evidence the brief points to directly. Report without editing. The review ends with this report; do not adjudicate findings, fix items, dispatch follow-up, or continue merely to obtain Pass.
|
|
136
|
+
|
|
137
|
+
A pass is valid. Use a high evidence bar, not a high item count. Do not broaden scope or treat possible improvement as unfinished work. Lead with the few items that can change the boundary judgment; prior review, urgency, implementation effort, or producer confidence does not establish correctness.
|
|
138
|
+
|
|
139
|
+
Read the brief's pointers: the agreement (through the Working Record's resume view when the brief names one), the docs, the Review Rules, and the diff. Then read the evidence as claims to check. Run a check listed under `May run`, in your own copy, only when that evidence is missing or does not convince you; run nothing else, and make no edits, network calls, or paid runs. When a claim still lacks evidence, report **Needs evidence** with the claim and required observing boundary. When you run a check, apply Verify Work's check-result and claim-result semantics; active evidence does not authorize editing or broaden the review.
|
|
140
|
+
|
|
141
|
+
Apply the lenses the brief names, or all of those under Apply The Lenses. Do not search for unrelated issues. Report an out-of-scope issue only when it materially affects whether this boundary may be crossed. Classify each item and determine the judgment as Review Work's SKILL.md does: "Classify Before Calling Something A Defect" and the independent judgment under "Perform A Selected Independent Review".
|
|
142
|
+
|
|
143
|
+
For Review 2 or 3, inspect only the diff since the previous review's reviewed state: accepted corrections, affected interactions, new evidence, and remaining risk. Never repeat a finding already Accepted, Rejected, or fixed; reopen a Rejected one only with contradictory evidence. For each new Blocking Issue, state whether it repeats, extends, invalidates, or exposes another consequence of a prior correction; is independent; or cannot yet be related from the evidence. Report a related pattern rather than proposing another patch. Review 3 is final; do not recommend Review 4.
|
|
144
|
+
|
|
145
|
+
The report is evidence for adjudication. It is not authority over source truth, owner decisions, corrections, or another independent dispatch. Pass, Non-blocking, Inconclusive, and Blocking are all valid exits. Write it in this shape, omitting empty groups and Improvements that are neither materially relevant nor requested:
|
|
146
|
+
|
|
147
|
+
```md
|
|
148
|
+
Review type: independent
|
|
149
|
+
Review number: [1 | 2 | 3]
|
|
150
|
+
Boundary: [reviewed work and protected future action]
|
|
151
|
+
Reviewed state: [state identity]
|
|
152
|
+
Judgment: Pass | Non-blocking | Inconclusive | Blocking
|
|
153
|
+
Reasoning: [concise evidence-backed judgment]
|
|
154
|
+
Relationship to prior items (Review 2 or 3 only): [related consequence | independent defect | unclear, with evidence]
|
|
155
|
+
|
|
156
|
+
### Review Items
|
|
157
|
+
|
|
158
|
+
#### Issues — Blocking
|
|
159
|
+
- [location, issue, violated source, evidence, consequence, correction constraints or owning activity]
|
|
160
|
+
|
|
161
|
+
#### Issues — Non-blocking
|
|
162
|
+
- [location, issue, evidence, why it can be deferred]
|
|
163
|
+
|
|
164
|
+
#### Unresolved — Questions
|
|
165
|
+
- [question, effect on judgment, required answer]
|
|
166
|
+
|
|
167
|
+
#### Unresolved — Needs evidence
|
|
168
|
+
- [claim, required observing boundary, available evidence and limit, why the gap affects judgment, smallest evidence that could disagree]
|
|
169
|
+
|
|
170
|
+
#### Pre-existing
|
|
171
|
+
- [location, defect, evidence that it predates the change, consequence]
|
|
172
|
+
|
|
173
|
+
#### Improvements
|
|
174
|
+
- [only when materially relevant or requested: improvement, benefit, evidence]
|
|
175
|
+
|
|
176
|
+
Evidence gaps: [unproved claims or none]
|
|
177
|
+
```
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Checks a work reviewer's pointer brief (references/work.md, "The Brief") before it is dispatched: every
|
|
3
|
+
// required labelled line is present, every backticked path it points to exists, and every commit it names resolves.
|
|
4
|
+
// It checks pointers only; whether they point to the right inputs is the initiator's judgment.
|
|
5
|
+
// Usage: check-brief.mjs <brief> [--root <repository>] [--json]
|
|
6
|
+
// Paths resolve against --root (default: the current directory); absolute paths are allowed, and a "#anchor" suffix is
|
|
7
|
+
// ignored. Exit code 0 when the brief passes, 1 when it has problems, 2 on a usage error.
|
|
8
|
+
import { execFileSync } from "node:child_process";
|
|
9
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
10
|
+
import { dirname, isAbsolute, resolve } from "node:path";
|
|
11
|
+
import { parseArgs } from "node:util";
|
|
12
|
+
|
|
13
|
+
const USAGE = "Usage: check-brief.mjs <brief> [--root <repository>] [--json]";
|
|
14
|
+
|
|
15
|
+
// Lines every brief carries. "paths" lines must point to at least one existing file unless "none" is allowed and given.
|
|
16
|
+
const REQUIRED = [
|
|
17
|
+
{ label: "Review", kind: "number" },
|
|
18
|
+
{ label: "Boundary", kind: "text" },
|
|
19
|
+
{ label: "Reviewed state", kind: "commit" },
|
|
20
|
+
{ label: "Method", kind: "paths" },
|
|
21
|
+
{ label: "Agreement", kind: "paths" },
|
|
22
|
+
{ label: "Docs", kind: "paths", none: true },
|
|
23
|
+
{ label: "Review rules", kind: "paths", none: true },
|
|
24
|
+
{ label: "Diff", kind: "range" },
|
|
25
|
+
{ label: "Evidence", kind: "paths", none: true },
|
|
26
|
+
{ label: "May run", kind: "text" },
|
|
27
|
+
{ label: "Report", kind: "report" },
|
|
28
|
+
];
|
|
29
|
+
const OPTIONAL = [
|
|
30
|
+
{ label: "Previous report", kind: "paths" },
|
|
31
|
+
{ label: "Lenses", kind: "text" },
|
|
32
|
+
{ label: "Notes", kind: "text" },
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
function fail(message) {
|
|
36
|
+
console.error(message);
|
|
37
|
+
process.exit(2);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let parsed;
|
|
41
|
+
try {
|
|
42
|
+
parsed = parseArgs({
|
|
43
|
+
allowPositionals: true,
|
|
44
|
+
options: { root: { type: "string" }, json: { type: "boolean", default: false } },
|
|
45
|
+
});
|
|
46
|
+
} catch (error) {
|
|
47
|
+
fail(`${error.message}\n${USAGE}`);
|
|
48
|
+
}
|
|
49
|
+
const { values, positionals } = parsed;
|
|
50
|
+
if (positionals.length !== 1) fail(USAGE);
|
|
51
|
+
const briefPath = resolve(positionals[0]);
|
|
52
|
+
if (!existsSync(briefPath)) fail(`no such brief: ${positionals[0]}`);
|
|
53
|
+
const root = resolve(values.root ?? ".");
|
|
54
|
+
|
|
55
|
+
// A labelled line is "Label: value" at the start of a line, optionally as a list item or in bold ("**Label:**").
|
|
56
|
+
const known = [...REQUIRED, ...OPTIONAL];
|
|
57
|
+
const lines = new Map();
|
|
58
|
+
for (const raw of readFileSync(briefPath, "utf8").split("\n")) {
|
|
59
|
+
const match = raw.match(/^\s*(?:[-*]\s+)?(?:\*\*)?([A-Za-z ]+?):(?:\*\*)?\s*(.*)$/);
|
|
60
|
+
if (!match) continue;
|
|
61
|
+
const spec = known.find((each) => each.label.toLowerCase() === match[1].trim().toLowerCase());
|
|
62
|
+
if (spec && !lines.has(spec.label)) lines.set(spec.label, match[2].trim());
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const problems = [];
|
|
66
|
+
const problem = (label, message) => problems.push({ line: label, problem: message });
|
|
67
|
+
|
|
68
|
+
const commitOf = (ref) => {
|
|
69
|
+
try {
|
|
70
|
+
return execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], {
|
|
71
|
+
cwd: root,
|
|
72
|
+
encoding: "utf8",
|
|
73
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
74
|
+
}).trim();
|
|
75
|
+
} catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const unquote = (value) => value.replace(/^`|`$/g, "").trim();
|
|
80
|
+
const pathsIn = (value) => [...value.matchAll(/`([^`]+)`/g)].map((match) => match[1].replace(/#.*$/, "").trim());
|
|
81
|
+
const at = (path) => (isAbsolute(path) ? path : resolve(root, path));
|
|
82
|
+
|
|
83
|
+
function check(spec, value) {
|
|
84
|
+
if (!value) return problem(spec.label, "is empty");
|
|
85
|
+
if (spec.none && /^none\.?$/i.test(value)) return;
|
|
86
|
+
if (spec.kind === "number") {
|
|
87
|
+
const round = Number(value.match(/\d+/)?.[0]);
|
|
88
|
+
if (![1, 2, 3].includes(round)) problem(spec.label, `must be 1, 2 or 3, not "${value}"`);
|
|
89
|
+
} else if (spec.kind === "commit") {
|
|
90
|
+
const ref = unquote(value.split(/\s+/)[0]);
|
|
91
|
+
if (!commitOf(ref)) problem(spec.label, `names no commit in ${root}: ${ref}`);
|
|
92
|
+
} else if (spec.kind === "range") {
|
|
93
|
+
const range = unquote(value.split(/\s+/)[0]);
|
|
94
|
+
const ends = range.split(/\.{2,3}/);
|
|
95
|
+
if (ends.length !== 2 || !ends[0] || !ends[1]) return problem(spec.label, `must be <base>..<head>, not "${value}"`);
|
|
96
|
+
for (const end of ends) if (!commitOf(end)) problem(spec.label, `names no commit in ${root}: ${end}`);
|
|
97
|
+
} else if (spec.kind === "paths") {
|
|
98
|
+
const paths = pathsIn(value);
|
|
99
|
+
if (!paths.length)
|
|
100
|
+
return problem(spec.label, `points to no path; write each path in backticks${spec.none ? ', or "none"' : ""}`);
|
|
101
|
+
for (const path of paths)
|
|
102
|
+
if (!existsSync(at(path))) problem(spec.label, `points to a path that does not exist: ${path}`);
|
|
103
|
+
} else if (spec.kind === "report") {
|
|
104
|
+
const [path] = pathsIn(value);
|
|
105
|
+
if (!path) return problem(spec.label, "names no report path in backticks");
|
|
106
|
+
if (!existsSync(dirname(at(path)))) problem(spec.label, `is in a folder that does not exist: ${dirname(path)}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (const spec of REQUIRED) {
|
|
111
|
+
if (!lines.has(spec.label)) problem(spec.label, "is missing");
|
|
112
|
+
else check(spec, lines.get(spec.label));
|
|
113
|
+
}
|
|
114
|
+
for (const spec of OPTIONAL) if (lines.has(spec.label)) check(spec, lines.get(spec.label));
|
|
115
|
+
const round = Number(lines.get("Review")?.match(/\d+/)?.[0]);
|
|
116
|
+
if ((round === 2 || round === 3) && !lines.has("Previous report"))
|
|
117
|
+
problem("Previous report", `is missing; Review ${round} needs the previous report and its adjudication`);
|
|
118
|
+
|
|
119
|
+
const ok = problems.length === 0;
|
|
120
|
+
if (values.json) {
|
|
121
|
+
console.log(JSON.stringify({ command: "check-brief", brief: briefPath, root, ok, problems }, null, 2));
|
|
122
|
+
} else if (ok) {
|
|
123
|
+
console.log(`Brief passes: ${positionals[0]}`);
|
|
124
|
+
} else {
|
|
125
|
+
console.log(`Brief refused: ${positionals[0]}`);
|
|
126
|
+
for (const each of problems) console.log(`- ${each.line}: ${each.problem}`);
|
|
127
|
+
}
|
|
128
|
+
process.exit(ok ? 0 : 1);
|