@contextrail/assess 0.0.0-stage → 1.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +82 -2
  3. package/assets/brand/icon-dark.png +0 -0
  4. package/assets/brand/icon-light.png +0 -0
  5. package/dist/assess.d.ts +109 -0
  6. package/dist/assess.js +1548 -0
  7. package/dist/assessment.d.ts +566 -0
  8. package/dist/assessment.js +404 -0
  9. package/dist/budget.d.ts +24 -0
  10. package/dist/budget.js +52 -0
  11. package/dist/candidate.d.ts +147 -0
  12. package/dist/candidate.js +321 -0
  13. package/dist/change-summary.d.ts +33 -0
  14. package/dist/change-summary.js +26 -0
  15. package/dist/cli.d.ts +2 -0
  16. package/dist/cli.js +406 -0
  17. package/dist/contracts.d.ts +99 -0
  18. package/dist/contracts.js +62 -0
  19. package/dist/diagnostics.d.ts +44 -0
  20. package/dist/diagnostics.js +412 -0
  21. package/dist/evidence.d.ts +73 -0
  22. package/dist/evidence.js +297 -0
  23. package/dist/feedback.d.ts +21 -0
  24. package/dist/feedback.js +140 -0
  25. package/dist/git.d.ts +40 -0
  26. package/dist/git.js +207 -0
  27. package/dist/github.d.ts +90 -0
  28. package/dist/github.js +1342 -0
  29. package/dist/index.d.ts +14 -0
  30. package/dist/index.js +14 -0
  31. package/dist/mcp.d.ts +28 -0
  32. package/dist/mcp.js +173 -0
  33. package/dist/pack.d.ts +160 -0
  34. package/dist/pack.js +377 -0
  35. package/dist/phases.d.ts +65 -0
  36. package/dist/phases.js +63 -0
  37. package/dist/planning.d.ts +64 -0
  38. package/dist/planning.js +135 -0
  39. package/dist/pr-context.d.ts +316 -0
  40. package/dist/pr-context.js +198 -0
  41. package/dist/prepare.d.ts +71 -0
  42. package/dist/prepare.js +335 -0
  43. package/dist/presentation.d.ts +309 -0
  44. package/dist/presentation.js +1026 -0
  45. package/dist/provider-diagnostics.d.ts +93 -0
  46. package/dist/provider-diagnostics.js +233 -0
  47. package/dist/providers.d.ts +45 -0
  48. package/dist/providers.js +291 -0
  49. package/dist/report-contract.d.ts +132 -0
  50. package/dist/report-contract.js +240 -0
  51. package/dist/results.d.ts +90 -0
  52. package/dist/results.js +128 -0
  53. package/dist/resume.d.ts +25 -0
  54. package/dist/resume.js +116 -0
  55. package/dist/source.d.ts +167 -0
  56. package/dist/source.js +675 -0
  57. package/dist/verification.d.ts +386 -0
  58. package/dist/verification.js +168 -0
  59. package/dist/version.d.ts +2 -0
  60. package/dist/version.js +9 -0
  61. package/docs/README.md +24 -0
  62. package/docs/action-reference.md +85 -0
  63. package/docs/architecture.md +156 -0
  64. package/docs/configuration.md +83 -0
  65. package/docs/contributing.md +46 -0
  66. package/docs/getting-started.md +92 -0
  67. package/docs/github-actions.md +141 -0
  68. package/docs/operations.md +203 -0
  69. package/docs/review-results.md +173 -0
  70. package/docs/security.md +64 -0
  71. package/docs/standards.md +62 -0
  72. package/examples/assess-pull-request.yml +109 -0
  73. package/package.json +56 -5
@@ -0,0 +1,203 @@
1
+ # Operate and troubleshoot reviews
2
+
3
+ [Documentation index](README.md) · [Assess overview](../README.md)
4
+
5
+ Use this page when a review is slow, incomplete, failed, or needs to be audited. For finding semantics, use [results](review-results.md).
6
+
7
+ ## Inspect evidence and resume
8
+
9
+ Each new review creates `.contextrail/assess/<run-id>/` in the reviewed repository:
10
+
11
+ | File | Use |
12
+ | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
13
+ | `diff.patch` | Ordinary unified Git diff for the fixed comparison; sensitive-path omissions are explicit. |
14
+ | `preparation.md` | Human entry point: compared commits, file statistics/list, governing scope and omissions. |
15
+ | `evidence.md` / `evidence-index.json` | Readable governing standards and their indexed read-tool references; supporting source remains available on demand. |
16
+ | `state.json` | Progressive scope, accepted results, gaps, effective settings and decision. |
17
+ | `publications.json` | Original pinned contexts. |
18
+ | `prepared.json` | Preparation state before model execution. |
19
+ | `evidence.json` | Retrieved evidence and provenance. |
20
+ | `diagnostics.jsonl` | Safe phase/call/tool metadata, validation failures, usage and stop reasons; no full conversation capture. |
21
+ | `phases/` | Versioned phase handoffs with input/output identities, assessment and evidence references. |
22
+ | `report.json` / `report.md` | Machine-readable and readable result with citations, scope, outcomes and reasons. |
23
+ | `manifest.json` | SHA-256 file digests binding the available finalized bundle. |
24
+
25
+ The `diff` field in JSON records is the initial paged tool view. `status: truncated`
26
+ means that view is bounded; `next` provides a continuation when available. It does not
27
+ indicate whether the separate `diff.patch` capture succeeded. Check `preparation.md`
28
+ for full permitted capture availability, exclusions and limits. Pinned source can be
29
+ read through tools; captured or available content is not proof it was examined.
30
+
31
+ Preparation produces state and publications; later files depend on reaching review
32
+ execution/finalization. Invalid startup settings may fail before any run directory
33
+ exists. SIGINT/SIGTERM and operational stops preserve accepted work where writable.
34
+ Resume a sealed incomplete run with the original comparison and settings:
35
+
36
+ ```sh
37
+ contextrail-assess run --reviewer skill://YOUR/REVIEWER --repo . --base BASE --head HEAD \
38
+ --provider openrouter --model YOUR_MODEL --resume .contextrail/assess/RUN_ID
39
+ ```
40
+
41
+ Resume a run from one process at a time. Resume validates the saved manifest, publications, source
42
+ identity and evidence before reusing work.
43
+ It uses the saved standards rather than fetching newer versions. Completed assessment work can
44
+ survive failed concern validation. Pending concerns remain unresolved; observations without nominated
45
+ concerns remain explicitly observed, not certified compliant. Policy determines the check outcome. Limits
46
+ apply to the new invocation and cumulative usage remains in the run. Hard termination can leave
47
+ an unsealed run, which resume refuses. Start a new review for different source or standards.
48
+
49
+ Review artifacts contain assessment bodies and necessary cited source evidence. Routine diagnostics
50
+ exclude full prompts/responses and source excerpts. New local run directories
51
+ and files use owner-only permissions. Runtime credential values are redacted, but
52
+ that does not make cited source suitable for public sharing. You control
53
+ artifact access and retention; remove a run when its accountability purpose ends.
54
+ Provider error bodies, HTTP headers and hidden reasoning are omitted. Reported
55
+ reasoning-token counts describe usage, not access to hidden reasoning.
56
+
57
+ ## Health, logging and cost
58
+
59
+ Use `--log-level debug` locally or `log-level: debug` in the Action for additional progress telemetry.
60
+ Stdout remains machine JSON; progress goes to stderr. Required diagnostic artifacts are retained at every level.
61
+
62
+ The summary distinguishes execution stop reason, assessment completeness, CLI exit and
63
+ report ingestion. It includes bounded finding/correction text, unresolved assessments,
64
+ exact comparison SHAs, package/reviewer identity, requested/returned models, runtime/SDK
65
+ versions, review duration, elapsed time before summary, calls/steps, and reported cost/token
66
+ usage. Links lead to the workflow and evidence artifact; full source citations and assessment records
67
+ stay in the artifact; full model conversations are not routinely captured. Model text is escaped to prevent injected Markdown, HTML and mentions.
68
+
69
+ Cost uses provider wire usage when present, including HTTP-200 calls rejected by the SDK;
70
+ SDK and wire records are never added together. Missing or partial per-call coverage is explicit,
71
+ not zero cost. Output tokens already include reasoning. Provider-reported cost is not a verified
72
+ invoice or total infrastructure cost. `received` confirms ingestion, not review approval.
73
+
74
+ Provider recovery records identify the phase, call number, retry count for that request,
75
+ and backoff in milliseconds. `provider_recovered` confirms a successful SDK response;
76
+ `provider_retry_exhausted` means all three attempts for that request failed. Recovery does not
77
+ mean the assessment itself passed. All requests consume the existing execution budget.
78
+ Each non-streaming provider attempt has a 180-second deadline. On expiry, the transport is
79
+ aborted and its rejection is awaited before the existing retry policy starts another attempt;
80
+ completed tools and accepted criteria are not replayed. Safe recovery diagnostics identify
81
+ `ProviderRequestTimeout`. There are still at most two retries per request, and each attempt
82
+ consumes the original call/time/token budget. Caller cancellation and the phase deadline take
83
+ precedence and do not trigger retries. A timeout may lack provider usage and must not be counted
84
+ as zero cost. This deadline is fixed; it does not extend the consuming workflow's overall limit.
85
+
86
+ Parser failures include a fixed `parse.code` such as `prototype_property_rejected`,
87
+ `unexpected_end`, `unterminated_string`, `invalid_control_character`, `unexpected_token`,
88
+ or the fallback `syntax_error`. `parse.bytes` records rejected JSON size when the SDK exposes
89
+ it, otherwise null. These categories distinguish observed parser failures without retaining
90
+ exception messages, response bodies, prompts, source or arbitrary provider fields. Unknown
91
+ syntax details remain unknown rather than being inferred from a generic exception name.
92
+
93
+ ### Diagnose a failed run
94
+
95
+ Start with the Action summary's terminal failure, then open `diagnostics.jsonl` in the
96
+ artifact. Correlate events by run ID, phase, attempt and call. The summary links the stop
97
+ reason to the last failing provider request in that phase; a recovered earlier error is
98
+ not the final cause. Updated summary behavior requires the new Action SHA as well as the
99
+ new npm package; changing `ASSESS_VERSION` alone does not update Action scripts.
100
+
101
+ At `info`, phase boundaries, provider request start/completion, assessment progress,
102
+ recovery, reporting and final outcome appear on stderr. `debug` also shows tool activity,
103
+ submission diagnostics and SDK steps. `quiet` retains artifacts and prints fatal failures.
104
+ All levels write the same bounded diagnostic artifact; stdout remains machine JSON.
105
+
106
+ Provider start/completion/failure events include request bytes, message and tool counts,
107
+ tool-schema bytes and the explicit output-token limit when available. These are wire
108
+ measurements, not token estimates. Request structure is inspected only up to 4 MiB;
109
+ above that size the byte count remains available but structural counts are unknown.
110
+ HTTP completion describes receipt and usage capture, not successful SDK parsing or a
111
+ valid review. Tool and phase terminal events record duration too.
112
+
113
+ | Failure category | What to check next | Automatic recovery |
114
+ | --------------------------------------------------------- | --------------------------------------------------------- | ---------------------------------------------------- |
115
+ | Context limit or oversized request | Request size and growth, evidence volume, provider limits | No identical retry for HTTP 400/413 |
116
+ | Invalid schema or unsupported parameter | Tool schema, provider/model support, configured effort | No identical retry for HTTP 400 |
117
+ | Credentials, permissions, credit limit, unavailable model | Selected account, access, quota and model ID | No retry for HTTP 401/402/403/404 |
118
+ | Rate limit, HTTP timeout or server failure | Provider health and retry events | Existing bounded retry for 408/429/5xx |
119
+ | Transport failure | Fixed DNS, connection, timeout or cancellation category | SDK-marked retryable transport failures only |
120
+ | Response parse/schema failure | Parser category and safe validation paths | Existing bounded retry; validation is never bypassed |
121
+ | Unknown request rejection | Status, call and safe classification evidence | Do not infer a cause from a generic HTTP 400 |
122
+
123
+ HTTP errors retain a fixed `details.code`/`reason`, an allowlisted parameter when supplied,
124
+ and whether classification came from an HTTP status, structured provider code or recognized
125
+ message pattern. Error-body inspection is capped at 64 KiB. `bodyFormat` marks missing,
126
+ oversized, non-JSON or unreadable bodies. Unrecognized details remain unknown; no arbitrary
127
+ provider message, body, header, source excerpt or prompt is written to routine diagnostics.
128
+ Nested OpenRouter upstream errors are inspected within the same bounds.
129
+
130
+ `finalization_failed` records the failed save/report/seal stage and original stop reason.
131
+ A secondary write failure does not replace the primary execution failure. Failure to persist
132
+ required diagnostic evidence prevents a clean completion. No automatic retry is added for
133
+ feedback or GitHub writes, which may already have taken effect.
134
+
135
+ ## Data handling and feedback
136
+
137
+ `--feedback send|off` defaults to **send**. After local finalization, `send` reports
138
+ only run/contract identity, origin and package/method version, exact reviewer/context
139
+ versions, expected scope, structured dispositions, adherence/confidence, finding
140
+ categories, policy/blocking decisions, completion/stop reason, and known duration,
141
+ provider/model/harness, call counts and provider-reported token usage with missingness.
142
+ A receipt confirms ingestion, not correctness.
143
+
144
+ Method `candidate-adherence-v2` reports observations and focused concern validation counts.
145
+ Observed rows do not claim verified compliance. A complete execution is
146
+ not a claim that external tests or deployment checks are complete. Out-of-scope operational
147
+ nominations remain in raw observations and decisions, not actionable PR follow-ups. The independently
148
+ checked change overview stays in local reports and PR output, outside the metadata feedback allowlist. Earlier reports retain their original method and interpretation.
149
+
150
+ Repository identity is withheld by default on both retrieval and review feedback.
151
+ `--repository-attribution` explicitly includes the Git-derived host, owner namespace
152
+ and repository name where available. It does not include the remote URL. `--feedback
153
+ off` disables review metadata submission; local reports are still written and ordinary
154
+ MCP retrieval still produces delivery events.
155
+
156
+ Review explanations, source/diffs, paths, quotations, prompts, responses, tool traces,
157
+ purpose, raw errors and audit files are not sent through review feedback. PR, CI and
158
+ artifact URLs are not reported. Detailed material stays in local artifacts and in
159
+ the requests to the model provider you selected; GitHub artifact storage is a
160
+ separate caller-controlled destination when you use the template.
161
+
162
+ ## Troubleshooting
163
+
164
+ - **Preparation fails:** run `prepare` with the same reviewer/repo/base/head.
165
+ Inspect `state.json` gaps and controlled diagnostics. Confirm the reviewer is
166
+ published, selects contexts, and its exact publications/prerequisites are readable
167
+ in the organization. Check MCP protocol, endpoint path and timeout.
168
+ - **No run directory:** check `--help`, Node/Git versions, required flags, resolvable
169
+ commits, MCP URL/token/org and write access. Configuration can fail before state
170
+ is created. Do not print credentials while checking whether they are exported.
171
+ - **Missing credentials:** export the MCP token and the selected provider key into
172
+ the actual CLI process. A `.env` file alone has no effect. Preparation needs no
173
+ provider key; a review missing its provider key records a provider failure after
174
+ successful preparation.
175
+ - **Provider errors:** inspect the controlled provider category/status, requested
176
+ model and stop reason in state/audit. Confirm account access, quota, tool support
177
+ and effort compatibility. Correct configuration before another paid attempt;
178
+ the CLI does not choose a replacement provider or model.
179
+ - **Incomplete review:** inspect unresolved/not-evaluated entries, preparation gaps,
180
+ operation limits and available source. Fetch missing Git objects
181
+ or supply a better bounded comparison, then run again. Advisory mode does not
182
+ resolve missing evidence. An exit-1 run can still be incomplete.
183
+ - **Feedback failed:** inspect `execution.reporting` or `reporting-status.json` when
184
+ present. Local results and exit remain usable; check MCP access and the saved
185
+ controlled code. A receipt is not a review endorsement.
186
+
187
+ Provider retries and submission correction have separate bounds; see [recovery and validation](architecture.md#recovery-and-validation).
188
+
189
+ Narrative correction is reported through `change_summary_rejected` (specific validation code,
190
+ attempt and retryability) and `change_summary_correction_failed` (whether completed criteria were
191
+ preserved). A missing narrative does not itself make an otherwise completed artifact review fail.
192
+
193
+ ## Observation-based reports
194
+
195
+ The `candidate-adherence-v2` method records immutable observation groups and focused concern decisions.
196
+ Deploy the API report-contract support for `observed`, nullable confidence and `concernValidation`
197
+ before enabling feedback from this reviewer. The published package export and consumer workflow
198
+ continue to use the existing CLI and Action inputs; no benchmark scripts are required at runtime.
199
+ Inspect `state.observations`, `state.concernDecisions`, and the hash-bound phase artifacts to distinguish
200
+ an assessment omission from a rejected concern, missing premise or execution failure. No-concern
201
+ observations remain unvalidated rather than receiving a fabricated compliance score. The reader
202
+ summary is separately checked; an unavailable summary is reported without discarding completed
203
+ standards judgments. Older judgment-based saved runs need a new run rather than silent conversion.
@@ -0,0 +1,173 @@
1
+ # Results and next actions
2
+
3
+ [Documentation index](README.md) · [Assess overview](../README.md)
4
+
5
+ Start with the outcome, findings and open questions. A successful execution, a policy-permitted
6
+ exit and a fully resolved assessment are different things.
7
+
8
+ ## What should I do with this result?
9
+
10
+ | What you see | What it means | Your next action |
11
+ | -------------------------------- | -------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
12
+ | Supported binding violation | Validation found a contradiction of an applicable standard | Inspect the cited rule and source; fix the artifact or challenge the assessment with concrete evidence |
13
+ | Guidance departure | The artifact departs from selected guidance | Consider the explanation and tradeoff; it is not a binding violation |
14
+ | Unresolved | A decisive artifact fact is unavailable, or the applicable guidance is ambiguous | Supply the missing source/context, investigate the implementation, or ask the guidance owner to clarify the rule |
15
+ | Observed / no concern identified | The assessor recorded an observation without a confirmed finding | Use it as a bounded observation; it is not independently verified whole-standard compliance |
16
+ | Not evaluated / execution failed | The assessment could not finish usable work | Inspect diagnostics and resolve the failure before relying on coverage |
17
+
18
+ A context can have both confirmed findings and unresolved questions. The unresolved status does not
19
+ hide the findings or prevent a binding violation from blocking under your policy. A missing fact is
20
+ not automatically a violation.
21
+
22
+ If a finding is wrong, explain the applicable exception, missing implementation path or contrary
23
+ source evidence. With PR context enabled, a later run can use that explanation to focus inspection.
24
+ A comment, dismissal or resolved thread does not by itself waive the standard or prove a fix.
25
+
26
+ ## What Assess judges
27
+
28
+ The subject is the proposed artifact: code, configuration, workflows and documentation. Supporting
29
+ source at the pinned revisions is available for inspection. The selected published standards define
30
+ what matters; a topic mentioned in a standard does not create a demand for exhaustive assurance.
31
+
32
+ For example, a workflow that bypasses a required gate is an artifact concern. Whether today's PR
33
+ checks are green is outside this assessment. A broken keyboard handler can contradict an interaction
34
+ standard; absence of a screenshot alone is not a defect. Runtime evidence may help establish an
35
+ artifact property, but Assess does not run your tests or certify deployment readiness.
36
+
37
+ Keep CI and deployment execution checks as their own gates. Findings and unresolved questions here
38
+ concern the proposed implementation, not proof that an operator performed a process.
39
+
40
+ ## Understand coverage and validation
41
+
42
+ Each enumerated context has its own recorded outcome and exact publication version. An incomplete
43
+ scope remains incomplete; the tool does not silently claim that missing contexts were assessed.
44
+
45
+ The assessor records observations, applicability reasoning, source evidence and possible concerns.
46
+ Each nominated concern receives a focused independent validation. Thus **20 standards assessed** and
47
+ **four concerns checked** describe different work; the other 16 may contain observations without
48
+ independent validation. Neither count proves that every obligation in a standard was discovered.
49
+
50
+ The detailed report and PR ledger use the same assessment labels. A recorded observation is not
51
+ “unexamined.” Truly missing or unfinished assessment work remains visible instead of being counted
52
+ as acceptance.
53
+
54
+ Current assessments do not assign a numeric confidence score. Observed and unresolved outcomes leave
55
+ scores null deliberately; reports say **not scored under this method**. Confirmed violations can carry
56
+ derived adherence `0`; that is an outcome encoding, not a percentage. Genuinely missing results remain
57
+ unknown. Citations establish provenance, not that the model's interpretation is correct.
58
+
59
+ ## Interpret judgments and exit codes
60
+
61
+ Choose `--enforcement advisory|required|strict`, or the Action's `enforcement` input. Default: `advisory`.
62
+
63
+ | Recorded outcome | advisory | required | strict |
64
+ | -------------------------------------------------------------------- | -------- | -------- | ------ |
65
+ | Supported binding violation | 0 | 1 | 1 |
66
+ | Unresolved evidence, without a supported violation | 0 | 0 | 1 |
67
+ | No binding violations or unresolved evidence | 0 | 0 | 0 |
68
+ | Execution failed or the required usable report could not be produced | 2 | 2 | 2 |
69
+
70
+ Guidance departures remain suggestions. An observed entry is not an evidence gap merely because it
71
+ lacks independent compliance certification. A finished assessment can leave a substantive question
72
+ unresolved; a timeout, invalid handoff or unfinished assessment is an execution failure.
73
+
74
+ - **0:** policy permits continuation; findings or questions may still exist in advisory mode.
75
+ - **1:** the selected policy requires attention.
76
+ - **2:** execution failed; inspect retained partial results and diagnostics.
77
+
78
+ Use branch protection to require a `required` or `strict` job when that is your team's policy. Leave
79
+ advisory optional if you do not want a merge gate. No mode changes which standards are assessed or
80
+ what evidence is retained.
81
+
82
+ The effective policy is saved with the run. The local `decision.reviewExit` preserves the policy
83
+ judgment while `decision.exitCode` reflects process failure if one occurs. A ContextRail feedback
84
+ submission failure is reported separately and does not erase the local result or change its exit.
85
+ See [operations](operations.md) if execution or feedback failed.
86
+
87
+ ## PR comments and inline corrections
88
+
89
+ Published packages display a 22px ContextRail symbol beside the heading, with light/dark variants
90
+ served from `assets/brand/` in the public Assess repository at the exact package release tag.
91
+ The exporter includes those assets before npm publication; no site login or alpha-site availability
92
+ is needed. Local monorepo builds keep the text heading because they have no corresponding public
93
+ release assets. Existing release tags must not be moved when changing the artwork.
94
+
95
+ The PR comment leads with the assessment outcome and enforcement mode, followed by a short
96
+ source-grounded change summary. Confirmed findings and unresolved artifact questions stay visible,
97
+ with corrections and pinned source links where available. A finished assessment with questions is distinct
98
+ from an execution failure; failures retain partial findings without presenting them as a complete assessment.
99
+
100
+ A compact coverage line stays visible. “No concern identified” describes an observation, not verified
101
+ compliance. **Standards and evidence** expands the per-criterion ledger, full unresolved explanations,
102
+ and additional human review focus. **Run details** holds cost, timing, identities and recent history.
103
+ The Action summary and artifact bundle retain the detailed assessment and operational diagnostics.
104
+ The comment omits the duplicate file inventory and does not infer resolved findings between runs.
105
+
106
+ The summary is reader orientation, separate from adherence and enforcement. Only an accepted,
107
+ source-checked narrative is displayed; its opening two sentences appear in the main comment.
108
+ Historical reports can retain human and operational follow-ups in the full report. Current
109
+ artifact-only reviews retain out-of-scope operational nominations in raw audit records without
110
+ turning them into developer action items.
111
+
112
+ Only verified binding violations are produced as eligible inline findings by this method; guidance remains in the summary. Historical reports may retain the older behavioral-defect category. The current assessor does not conduct unrelated bug hunting.
113
+
114
+ Set `inline-comments: 'true'` alongside `post-comment: 'true'` to publish verified
115
+ findings on exact diff lines. Eligible corrections include GitHub suggestion blocks that
116
+ the developer can apply. Suggestions require independently accepted correction evidence,
117
+ an exact matching head range and valid diff anchor; source inspection is not test execution.
118
+ Uncertain, overlapping, deletion-only or multifile corrections remain explanatory findings.
119
+ Nothing applies patches or commits automatically.
120
+
121
+ Inline publication uses COMMENT reviews, retains existing threads and deduplicates matching
122
+ findings. It does not approve, request changes, resolve threads or treat a human resolution
123
+ as proof of a fix. Incomplete duplicate lookup stops new inline writes. Unanchorable findings,
124
+ rejected batches and items beyond the bounded inline batch remain in the main report.
125
+ A superseded comparison is attributed to its reviewed commit, not claimed as current.
126
+
127
+ Reusable CLI adapters are `contextrail-assess github prepare` and `github publish`;
128
+ see `--help` for required paths/options. `github publish --dry-run` creates inspectable
129
+ payloads without network writes. Provider credentials do not reach these adapter commands,
130
+ and GitHub write tools are never exposed to models.
131
+
132
+ The full change inventory remains in the audit report rather than the PR comment.
133
+
134
+ ## Review history and conversational context
135
+
136
+ The bot comment keeps a versioned, bounded state record in an HTML comment and a visible
137
+ history table showing the latest three runs in collapsed run details. The hidden state retains ten reported runs with GitHub run/attempt links, unique CLI run IDs, exact work
138
+ revisions, requested model, package/reviewer versions, execution/completion, finding/unresolved
139
+ counts, duration and reported cost. Rerunning the same attempt replaces its row; later attempts
140
+ add rows. A failed provider run does not erase the last finished assessment. That snapshot
141
+ retains up to 30 criterion identities/publication versions and short findings/unresolved reasons.
142
+ Older rows, skipped/comment-write failures and runs before adoption may be absent; the table
143
+ links to GitHub's full workflow history. This is a convenience cache, not an exhaustive ledger.
144
+
145
+ Hidden comments are readable, mutable PR content, not private storage, a signature or proof
146
+ that a review passed. The Action reads only the matching bot-owned record, validates its
147
+ version/repository/PR/reviewer and bounds its contents. Unsupported or corrupt state is treated
148
+ as unavailable rather than trusted. PR comments persist independently of artifact retention.
149
+
150
+ With `pr-context: 'true'`, a trusted read-only step collects recent 20 issue comments, 10 submitted
151
+ reviews (including dismissed states), 10 dismissal events with reasons, and 10 native review threads with five replies each,
152
+ including resolution/outdated state and authorship. At most 80 excerpts are supplied, with bodies capped at 1000 characters.
153
+ The matching CLI's `--pr-context PATH` reads a schema-validated file capped at 200 KiB, bound to
154
+ the current base/head/reviewer and reported GitHub repository. Prior assessment/head identities,
155
+ criterion versions and up to 100 changed paths since that head are included when available.
156
+ Missing history, pagination/truncation, inaccessible revisions and non-forward/force-push
157
+ comparisons are explicit. There is no whole-repository snapshot mode or automatic new review trigger.
158
+
159
+ Context is passed as a separate untrusted data field, never as caller purpose or governing
160
+ instructions. Native approval, dismissal or thread resolution does not waive a Standard or
161
+ prove a fix. The reviewer still accounts for every current criterion using current pinned
162
+ source/publications; historical outcomes never pre-populate current results or change exit policy.
163
+ Developer feedback can focus inspection and explain an addressed finding only with fresh evidence.
164
+ Work, guidance and model revisions remain distinct; changed judgment alone does not establish drift.
165
+ No tools for changing code, resolving threads or approving a PR are added.
166
+
167
+ Opting in sends these PR excerpts and historical judgments to the selected model. GitHub tokens
168
+ stay in trusted Action steps; known credentials are redacted before the model handoff. The raw
169
+ collector file is not uploaded. A validated/redacted context snapshot and hash accompany the
170
+ local run/audit; conversation bodies are excluded from the minimal ContextRail report projection.
171
+ The health summary identifies supplied discussion/history coverage. Keep the Action SHA and package version qualified together when upgrading.
172
+
173
+ For timing, cost, logs and evidence files, see [operations](operations.md).
@@ -0,0 +1,64 @@
1
+ # Security and data handling
2
+
3
+ [Documentation index](README.md) · [Assess overview](../README.md)
4
+
5
+ For maintainers and security reviewers deciding where Assess may run and which services may receive
6
+ its inputs. Assess runs in your local or CI environment, but assessment requires requests to your
7
+ selected model provider.
8
+
9
+ ## Where data goes
10
+
11
+ | Destination | Material |
12
+ | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
13
+ | ContextRail MCP | Guidance retrieval requests, organization scope and authorized authentication; optional repository attribution |
14
+ | Selected OpenAI/OpenRouter provider | Guidance, source excerpts and other assessment inputs; optional PR discussion/history when enabled |
15
+ | Local run directory | Pinned comparison, guidance, citations, observations, decisions, reports and bounded diagnostics |
16
+ | GitHub, when enabled | PR summary/inline comments and uploaded evidence artifacts, under the consumer's permissions and retention settings |
17
+ | ContextRail assessment feedback | Allowlisted run/standard identities and versions, structured outcomes, policy/completion and known telemetry—not source, quotations or conversations |
18
+
19
+ “No detailed evidence upload to ContextRail” does not mean “no third-party processing.” Choose a
20
+ provider account and data-handling arrangement appropriate for your source and guidance. A provider's
21
+ reported token usage or cost is not a verified invoice.
22
+
23
+ `--feedback send` is the default; `--feedback off` disables assessment-metadata submission.
24
+ Ordinary MCP retrieval still produces delivery events. Repository identity is withheld by default;
25
+ `--repository-attribution` includes the Git-derived host, owner namespace and repository name, not a
26
+ remote URL. An ingestion receipt confirms receipt, not correctness or compliance.
27
+
28
+ ## What source the model can inspect
29
+
30
+ The package fixes the base/head commits and reads source through bounded tools. Supporting unchanged
31
+ files at those snapshots are available when needed. It does not execute project scripts, install
32
+ project dependencies, run tests, initialize submodules or give the model a shell.
33
+
34
+ Conventional credential paths are excluded, and binary/submodule contents have explicit limits.
35
+ These protections are not a guarantee that arbitrary secrets embedded in ordinary source will be
36
+ detected. Review the repository's suitability before sending it to a provider.
37
+
38
+ PR text and prior comments are untrusted context, not governing instructions. Opting into PR history
39
+ sends bounded excerpts to the selected provider. See [PR context](review-results.md#review-history-and-conversational-context).
40
+
41
+ ## Credentials and execution trust
42
+
43
+ Use process environment or a secret manager locally, and named GitHub secrets in CI. Do not put keys
44
+ in CLI arguments, source, PR discussion or retained artifacts. The CLI does not load `.env` files.
45
+
46
+ The Action isolates installation, GitHub reads and model execution credentials between steps, but a
47
+ composite Action shares its caller's job. Never execute untrusted contribution code in that credentialed
48
+ job. Pin a reviewed Action commit and an exact qualified npm version; disabling install scripts does
49
+ not make an untrusted runtime package safe. The [GitHub guide](github-actions.md#triggers-permissions-and-trust)
50
+ explains the caller's responsibilities.
51
+
52
+ ## Retention and interpretation
53
+
54
+ The consumer controls local artifacts and GitHub access/retention. Action artifacts default to seven
55
+ days; choose an appropriate retention period rather than assuming a PR comment is a permanent audit
56
+ store. Comments can outlive artifacts and their hidden history is mutable convenience data.
57
+
58
+ The bundle records exact work/guidance identities and evidence. It is not independently signed
59
+ assurance, and no finding-free result certifies comprehensive compliance. Publication attribution can
60
+ be looked up in ContextRail using the recorded context/version; the package does not add an approval
61
+ workflow or currently embed publisher metadata in the evidence bundle.
62
+
63
+ See [operations](operations.md#data-handling-and-feedback) for the precise feedback boundary and
64
+ [Action reference](action-reference.md) for artifact delivery, cancellation and access behavior.
@@ -0,0 +1,62 @@
1
+ # Set up the standards to assess
2
+
3
+ [Documentation index](README.md) · [Assess overview](../README.md)
4
+
5
+ Use this guide with the subject-matter expert who owns your guidance and the maintainer connecting
6
+ Assess. These contexts belong to your organization’s shared guidance: connected tools can also
7
+ retrieve them while people create work. The reviewer skill selects which guidance Assess should
8
+ consider for a proposed change. Installing the package alone does not create an assessment scope.
9
+
10
+ ## Prepare the ContextRail inputs
11
+
12
+ 1. **Publish the guidance you want applied.** In ContextRail, contexts describe your organization's
13
+ Standards and Guidance. The owner maintains their meaning and publishes a version. An unpublished
14
+ draft is not the version Assess uses.
15
+ 2. **Publish a reviewer skill that selects those contexts.** A reviewer skill supplies assessment
16
+ instructions and an explicit context/domain selection. Its required context prerequisites are
17
+ also resolved. An instruction-only skill has no assessable context set for this package.
18
+ 3. **Give the maintainer the published skill ID.** Use its `skill://…` identifier, not its title or a
19
+ run ID. The package option remains `--reviewer` because it selects this platform skill.
20
+ 4. **Provide authorized MCP access.** Your ContextRail administrator supplies the endpoint, organization
21
+ ID and a token authorized to read the selected skill and contexts. Feedback submission also needs
22
+ the corresponding access when enabled. Transfer the token through your secret manager, not a PR.
23
+ 5. **Run `prepare` before the first paid assessment.** Confirm the enumerated contexts and versions in
24
+ `preparation.md`. Missing or inaccessible guidance makes preparation incomplete rather than silently
25
+ reducing the scope.
26
+
27
+ [The first-assessment guide](getting-started.md) has the commands. [GitHub setup](github-actions.md)
28
+ shows where to store the connection values for CI.
29
+
30
+ ## Write guidance that can be applied to an artifact
31
+
32
+ A useful standard identifies the behavior it governs, the requirement, and meaningful conditions
33
+ or permitted exceptions. Examples help the assessor distinguish an actual contradiction from a
34
+ preferred implementation.
35
+
36
+ For example:
37
+
38
+ > Release workflows must require the maintained quality gate before deployment. Emergency release
39
+ > paths must preserve the documented authorization and recovery requirements.
40
+
41
+ For a proposed workflow change, Assess can inspect dependencies, conditions and the emergency path.
42
+ “Make sure today's checks passed” asks about a run rather than the proposed artifact. Keep execution
43
+ checks in CI; use guidance to describe what those checks and delivery mechanisms must enforce.
44
+
45
+ Avoid making the reviewer skill a second source of company requirements. Put the governing rules in
46
+ selected contexts. The skill can focus investigation, but its instructions do not expand this package
47
+ into a general operational audit or unrelated bug hunt.
48
+
49
+ ## Keep ownership after publication
50
+
51
+ Assess pins the published skill and context versions for each run. Later publications affect later
52
+ runs, not the meaning of an existing record. The platform's context/version reference lets your team
53
+ return to the publication it used.
54
+
55
+ A finding is a proposed judgment about the artifact, not an automatic change to your standard.
56
+ Ambiguous guidance can remain unresolved. Use that evidence and the structured feedback in ContextRail
57
+ to decide whether the code, the guidance or the assessment needs correction. Experts retain publication
58
+ authority; developer inaction does not automatically retire a standard.
59
+
60
+ Every enumerated context receives a status, including unfinished work. That accounting is not a claim
61
+ that every obligation in a long standard was discovered or independently verified. See
62
+ [how to interpret results](review-results.md).
@@ -0,0 +1,109 @@
1
+ # Retained direct CLI example; select a qualified published version before enabling.
2
+ # The public export fills ASSESS_PACKAGE_VERSION with its selected version.
3
+ # Licensed under Apache-2.0; confirm publication before enabling this credentialed workflow.
4
+ name: Assess against organizational guidance
5
+
6
+ on:
7
+ pull_request_target:
8
+ types: [opened, synchronize, reopened]
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ assess:
15
+ # Example consumer policy: review same-repository human PRs on this event.
16
+ # A skipped job is not evidence that the change passed review.
17
+ if: >-
18
+ github.event_name == 'pull_request_target' &&
19
+ github.event.pull_request.head.repo.full_name == github.repository &&
20
+ github.event.pull_request.user.type == 'User'
21
+ runs-on: ubuntu-latest
22
+ timeout-minutes: 30
23
+ env:
24
+ ASSESS_PACKAGE_VERSION: '1.0.0-alpha.1'
25
+ REVIEWER_ID: skill://your-domain/your-reviewer
26
+ ASSESS_PROVIDER: openai
27
+ ASSESS_MODEL: REPLACE_WITH_PROVIDER_MODEL_ID
28
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
29
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
30
+ steps:
31
+ - name: Set up Node
32
+ uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
33
+ with:
34
+ node-version: '22.18.0'
35
+ package-manager-cache: false
36
+ token: ''
37
+
38
+ - name: Install ContextRail Assess outside the assessed repository
39
+ working-directory: ${{ runner.temp }}
40
+ shell: bash
41
+ run: |
42
+ mkdir -p "$RUNNER_TEMP/contextrail-tool"
43
+ npm install --prefix "$RUNNER_TEMP/contextrail-tool" \
44
+ --userconfig /dev/null --registry https://registry.npmjs.org \
45
+ --ignore-scripts --no-audit --no-fund --package-lock=false \
46
+ "@contextrail/assess@$ASSESS_PACKAGE_VERSION"
47
+
48
+ - name: Fetch the fixed comparison without checking out contribution files
49
+ shell: bash
50
+ env:
51
+ GH_READ_TOKEN: ${{ github.token }}
52
+ SOURCE_REPOSITORY: ${{ github.repository }}
53
+ SOURCE_SERVER: ${{ github.server_url }}
54
+ run: |
55
+ [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ && "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]
56
+ [[ "$SOURCE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]
57
+ [[ "$SOURCE_SERVER" == 'https://github.com' ]]
58
+ git init --template= "$GITHUB_WORKSPACE/assess-source"
59
+ git -C "$GITHUB_WORKSPACE/assess-source" remote add origin \
60
+ "$SOURCE_SERVER/$SOURCE_REPOSITORY.git"
61
+ auth=$(printf 'x-access-token:%s' "$GH_READ_TOKEN" | base64 -w0)
62
+ # No depth/filter: both complete commit histories and supporting blobs are available.
63
+ git -c core.hooksPath=/dev/null -c core.fsmonitor=false \
64
+ -c "http.extraHeader=AUTHORIZATION: basic $auth" \
65
+ -C "$GITHUB_WORKSPACE/assess-source" fetch --no-tags origin "$BASE_SHA" "$HEAD_SHA"
66
+ git -C "$GITHUB_WORKSPACE/assess-source" cat-file -e "$BASE_SHA^{commit}"
67
+ git -C "$GITHUB_WORKSPACE/assess-source" cat-file -e "$HEAD_SHA^{commit}"
68
+
69
+ - name: Assess the fixed commits
70
+ id: assess
71
+ working-directory: ${{ runner.temp }}
72
+ shell: bash
73
+ env:
74
+ CONTEXTRAIL_MCP_URL: ${{ vars.CONTEXTRAIL_MCP_URL }}
75
+ CONTEXTRAIL_MCP_ORG_ID: ${{ vars.CONTEXTRAIL_MCP_ORG_ID }}
76
+ CONTEXTRAIL_MCP_AUTH_TOKEN: ${{ secrets.CONTEXTRAIL_MCP_AUTH_TOKEN }}
77
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
78
+ # For OpenRouter, replace the OpenAI entry with this secret and change provider/model.
79
+ # OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
80
+ run: |
81
+ # exec preserves 0 (accepted), 1 (blocker), or 2 (incomplete/error).
82
+ exec "$RUNNER_TEMP/contextrail-tool/node_modules/.bin/contextrail-assess" run \
83
+ --reviewer "$REVIEWER_ID" --provider "$ASSESS_PROVIDER" --model "$ASSESS_MODEL" \
84
+ --repo "$GITHUB_WORKSPACE/assess-source" --base "$BASE_SHA" --head "$HEAD_SHA" \
85
+ --feedback send
86
+
87
+ - name: Locate available assessment files
88
+ id: files
89
+ if: ${{ always() }}
90
+ shell: bash
91
+ run: |
92
+ directory="$GITHUB_WORKSPACE/assess-source/.contextrail/assess"
93
+ if [[ -d "$directory" ]] && [[ -n "$(find "$directory" -type f -print -quit)" ]]; then
94
+ echo 'available=true' >> "$GITHUB_OUTPUT"
95
+ else
96
+ echo 'available=false' >> "$GITHUB_OUTPUT"
97
+ echo 'No assessment files are available. Inspect setup and Assess logs.'
98
+ fi
99
+
100
+ - name: Archive available reports and audit
101
+ if: ${{ always() && steps.files.outputs.available == 'true' }}
102
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
103
+ with:
104
+ name: contextrail-assess-${{ github.run_id }}-${{ github.run_attempt }}
105
+ path: assess-source/.contextrail/assess/
106
+ include-hidden-files: true
107
+ if-no-files-found: error
108
+ retention-days: 7
109
+ # Audits can contain source and model conversations; restrict artifact access.