devmethod-ai 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/devmethod-review/SKILL.md +1 -1
- package/.agents/skills/project-foundation/assets/START_HERE.md +2 -0
- package/.agents/skills/scoped-delivery/assets/REVIEW.md +2 -2
- package/.agents/skills/scoped-delivery/references/review-report.md +16 -0
- package/.agents/skills/scoped-delivery/references/review-workflow.md +10 -1
- package/.agents/skills/scoped-delivery/references/verification-and-cost.md +2 -0
- package/COMPATIBILITY.md +2 -0
- package/README.md +13 -9
- package/START_HERE.md +2 -0
- package/dist/doctor.js +3 -1
- package/dist/init.js +13 -0
- package/dist/review-agent.js +24 -0
- package/dist/review-runtime.js +9 -0
- package/docs/ADR-010-installed-review-renderer.md +9 -0
- package/docs/COMMANDS.md +2 -2
- package/docs/EVALUATION.md +4 -0
- package/docs/RELEASE-0.4.1.md +13 -0
- package/docs/REVIEW-GUIDE.md +5 -1
- package/docs/REVIEWS.md +6 -2
- package/evaluation/review-detection/README.md +30 -0
- package/evaluation/review-detection/fixtures/compatibility.mjs +4 -0
- package/evaluation/review-detection/fixtures/consumer.mjs +3 -0
- package/evaluation/review-detection/fixtures/control.mjs +5 -0
- package/evaluation/review-detection/fixtures/sensitive.mjs +11 -0
- package/evaluation/review-detection/fixtures/submission.mjs +8 -0
- package/evaluation/review-detection/observed-0.4.1/README.md +16 -0
- package/evaluation/review-detection/observed-0.4.1/TASK.md +1 -0
- package/evaluation/review-detection/observed-0.4.1/adjudication.json +65 -0
- package/evaluation/review-detection/observed-0.4.1/case-a/sensitive.mjs +10 -0
- package/evaluation/review-detection/observed-0.4.1/case-b/submission.mjs +7 -0
- package/evaluation/review-detection/observed-0.4.1/case-c/compatibility.mjs +3 -0
- package/evaluation/review-detection/observed-0.4.1/case-c/consumer.mjs +2 -0
- package/evaluation/review-detection/observed-0.4.1/case-d/control.mjs +5 -0
- package/evaluation/review-detection/observed-0.4.1/input-hashes.json +7 -0
- package/evaluation/review-detection/observed-0.4.1/method-hashes.json +76 -0
- package/evaluation/review-detection/observed-0.4.1/probe-results.json +58 -0
- package/evaluation/review-detection/observed-0.4.1/raw-findings.md +75 -0
- package/evaluation/review-detection/observed-0.4.1/review-probes.mjs +41 -0
- package/evaluation/review-detection/observed-0.4.1/score.json +17 -0
- package/evaluation/review-detection/oracle.json +9 -0
- package/evaluation/review-detection/reproduce.test.mjs +53 -0
- package/evaluation/review-detection/score.mjs +26 -0
- package/package.json +1 -1
- package/scripts/package-smoke.mjs +7 -2
|
@@ -13,6 +13,6 @@ Use [scoped-delivery](../scoped-delivery/SKILL.md) for this stage’s detailed p
|
|
|
13
13
|
|
|
14
14
|
Follow [the review workflow](../scoped-delivery/references/review-workflow.md): inspect the actual diff and relevant interactions, execute applicable checks, and report findings with locations, impact, evidence and proposed corrections. Distinguish checked, failed and uninspected scope. Do not modify product code unless corrections were requested. Review artifacts may be written as part of the requested review.
|
|
15
15
|
|
|
16
|
-
For a substantial review, use [the review record](../scoped-delivery/assets/REVIEW.md).
|
|
16
|
+
For a substantial review, use [the review record](../scoped-delivery/assets/REVIEW.md) and complete [the report delivery procedure](../scoped-delivery/references/review-report.md). Produce the real JSON, derived Markdown and interactive HTML as one flow; open the HTML when the user asks to view/open the report. A request such as “review these changes, then open the report” authorizes generation and local opening, with no extra confirmation. Keep a small review concise unless a report is requested. Never substitute fictional demo data for actual findings or ask the user to run a terminal command.
|
|
17
17
|
|
|
18
18
|
Return the concrete outcome, verification limits and one next command from the stage contract.
|
|
@@ -24,3 +24,5 @@ Examples:
|
|
|
24
24
|
This kit reduces repetitive framing; it does not by itself prove application quality or production readiness.
|
|
25
25
|
|
|
26
26
|
For a new product, `explore` researches existing solutions and discusses the findings before framing. `architecture` discusses structural options before dependent detail; `plan` discusses delivery scope and stays conditional on open decisions. Keep accepted choices, delegations and approved design. For substantial work without conventions, propose a mission PLAN with tickets and a dated REPRISE; keep existing single-file missions usable.
|
|
27
|
+
|
|
28
|
+
To review and view results in one request, use `$devmethod-review the current changes, then open the report` (Codex) or `/devmethod-review` with the same request (Claude Code/Cursor). The agent performs the review and generates/opens the real report using the renderer installed with scoped-delivery. No user-run npx command is needed.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Structured review authoring
|
|
2
2
|
|
|
3
|
-
Reuse the existing review convention, or docs/missions/<mission-id>/reviews/<review-id>/review.json. The JSON owns results.
|
|
3
|
+
Reuse the existing review convention, or docs/missions/<mission-id>/reviews/<review-id>/review.json. The JSON owns results. Keep small reviews in the conversation unless a report is requested. For substantial reviews or a requested report, follow [the installed format](../references/review-format.md), then [generate and deliver the report](../references/review-report.md) using the installed renderer. The agent performs these steps; do not hand terminal instructions back to the user.
|
|
4
4
|
|
|
5
|
-
Begin
|
|
5
|
+
Begin with the actual scope and sources. Replace the placeholders below and add only observed checks/findings/evidence. Generate Markdown and HTML from that record. An unavailable runtime or failed browser opening is a specific delivery limitation to report after preserving completed artifacts; never substitute a demo or describe an unexecuted check as passed.
|
|
6
6
|
|
|
7
7
|
```json
|
|
8
8
|
{
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Deliver the real review report
|
|
2
|
+
|
|
3
|
+
This is the final part of the review workflow, performed by the agent. First inspect actual work and record the checks, findings and limits using [the review format](review-format.md). A substantial review produces JSON, Markdown and HTML; a small review may stay in the conversation unless a report is requested. Use the actual inspected revision, and label synthetic test inputs only as test fixtures.
|
|
4
|
+
|
|
5
|
+
1. Write the real review JSON in the project's existing convention, or docs/missions/<mission-id>/reviews/<review-id>/review.json. Preserve previous review snapshots. Do not fabricate checks to make the report look complete.
|
|
6
|
+
2. Resolve the installed scoped-delivery folder from this skill, even if renamed. Its `scripts/review-agent.mjs` renderer ships with the installation and needs Node.js 22+, but no npm, npx, server or network. It works in ESM, CommonJS and projects without package.json. Execute it yourself, with separately quoted arguments, for example from the project root:
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
node .agents/skills/scoped-delivery/scripts/review-agent.mjs --review docs/missions/my-mission/reviews/review-1/review.json --output docs/missions/my-mission/reviews/review-1/review.html --markdown docs/missions/my-mission/reviews/review-1/REVIEW.md --open
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Replace sample paths with the actual record and fresh output paths. Use an absolute script path if needed and `--dest` for the actual project root. Omit `--open` for a report-only request or a headless environment. A request to open/show the report already authorizes local browser opening. Do not add a second approval or show this command as work for the user to perform.
|
|
13
|
+
3. The renderer validates the real record before writing. Correct schema problems from actual evidence, preserving all review outcomes; never remove findings or change results to pass validation. Existing outputs are protected: select a new snapshot path. A valid rendering is not a passed code review.
|
|
14
|
+
4. Check that both output files exist and report the review conclusion, unresolved findings, coverage limits and clickable artifact paths. If opening fails, retain and link the generated files, explain that specific limitation and use an available host file/browser preview where supported. Do not claim the browser opened merely because generation passed.
|
|
15
|
+
|
|
16
|
+
The installed helper intentionally has no demo mode. For a legacy installation lacking the helper, use an available reviewed DevMethod CLI/source checkout internally; otherwise stage the published kit through the project's authorized installation procedure and use its renderer without overwriting the project. Never stop at telling the user to run npx. If Node.js, download access or browser support is genuinely unavailable, finish independent inspection and preserve the real record, naming the specific remaining export/opening blocker. An actual environment restriction is not permission to bypass it.
|
|
@@ -10,11 +10,20 @@ Consult relevant official documentation and search for skills published by the a
|
|
|
10
10
|
|
|
11
11
|
Choose checks according to affected scope: need/acceptance, business rules/edges, architecture/contracts, authorization/data isolation, integrity/transactions/concurrency, errors/recovery, UX/accessibility/visual fidelity, performance/operations/maintainability and relevant tests. Read affected interactions, then execute available checks. For UI inspect real rendering and interactions at relevant viewports even when automated tests pass; for backend inspect flows, errors, contracts and permissions.
|
|
12
12
|
|
|
13
|
+
Follow relevant behavior beyond changed lines. Trace affected callers, other user journeys, downstream contract consumers and previously stored data. Identify the concrete compatibility obligation before proposing a migration. A clean diff or passing new-path test does not establish compatibility with existing consumers or records.
|
|
14
|
+
|
|
15
|
+
Select the following probes only when the change's data and failure risks justify them; a small text or isolated logic change does not require a full audit:
|
|
16
|
+
|
|
17
|
+
- For credentials, tokens, secrets or personal data, follow input/storage through transformations and error handling to each reachable sink: logs, exception messages, API responses and telemetry. Check success and failure paths, including SDK/HTTP error objects and nested fields. Use synthetic sentinel values, captured outputs and relevant configuration to reproduce exposure without collecting real secrets. State which sinks were inspected and which remain unknown. Masking secrets in the review report protects that artifact; it does **not** demonstrate that the application prevents leaks. Inspect the original synthetic output before report sanitization and record the application-level result separately.
|
|
18
|
+
- For mutable state or external operations, exercise relevant concurrency, double submission, timeouts, retries, interruption and partial success. Identify the invariant, ordering, commit point and retry/recovery behavior. A timed-out call may already have applied its effect. Inspect idempotency scope, atomicity and compensation/reconciliation where needed; use controlled interleavings or injected failures rather than relying on a race happening by chance. Explain the observable integrity or recovery consequence, not merely that a transaction or retry mechanism is absent.
|
|
19
|
+
|
|
20
|
+
Classify each criticism explicitly in the existing finding rationale: a confirmed defect with reproduction/evidence; a risk requiring a named verification step; a project requirement linked to its authoritative source; or a style preference. A requirement is a basis for assessment, not itself proof of a violation. Personal style preferences are non-blocking and should normally stay out of defect findings. An architecture criticism must demonstrate a concrete consequence for a relevant scenario (such as data loss, incompatible consumers or unrecoverable partial success); disagreement with a pattern or layering preference alone is insufficient. Severity follows justified impact and the project's blocking policy, not the reviewer's preferred design.
|
|
21
|
+
|
|
13
22
|
Keep checks distinct from findings. Record passed, failed, not-run, blocked and out-of-scope checks with result, evidence, inspected revision, kind (automated/manual) and non-execution reason. A failed test count is not a finding count. Distinguish confirmed problems from plausible risks still needing verification. Severity expresses justified impact; confidence expresses evidence. Absence of findings is not complete coverage.
|
|
14
23
|
|
|
15
24
|
Each finding needs a stable ID, concrete title/domain, justified severity, confidence, resolution state, precise location, trigger, expected/observed behavior, impact, evidence or reproduction, proposed correction and trade-offs, relevant sources/tickets and resolution verification. Closing a pane, accepting a risk or applying a patch does not prove resolution: retain history and mark resolved only with fresh relevant verification.
|
|
16
25
|
|
|
17
|
-
Use [structured review authoring](../assets/REVIEW.md) for the versioned record and
|
|
26
|
+
Use [structured review authoring](../assets/REVIEW.md) for the versioned record and [report delivery](review-report.md) to generate Markdown/HTML and open the report when requested. Carry the request through these steps instead of asking the user to run npx.
|
|
18
27
|
|
|
19
28
|
## Communicate and preserve ownership
|
|
20
29
|
|
|
@@ -17,6 +17,8 @@ Use project gates even when stricter. Do not invent an unavailable command; repo
|
|
|
17
17
|
|
|
18
18
|
Connect every finding to a location, observable consequence, scenario, and correction. Distinguish bug, demonstrated risk, and preference. Do not request multiple identical opinions to create an appearance of certainty. If independent review is required but impossible, report it rather than simulating it.
|
|
19
19
|
|
|
20
|
+
Use [the review workflow](review-workflow.md) as the owner of data-flow, failure-scenario, compatibility and finding-classification guidance. Choose relevant probes; do not turn every small change into an exhaustive audit.
|
|
21
|
+
|
|
20
22
|
Evaluate complete diff, boundaries, behavior, security, and tests at the stated commit. Re-examine touched areas after corrections, and the whole only when impact warrants it.
|
|
21
23
|
|
|
22
24
|
## Operational cost
|
package/COMPATIBILITY.md
CHANGED
|
@@ -26,3 +26,5 @@ Run separately in authenticated Codex, Claude Code and Cursor Agent sessions. Us
|
|
|
26
26
|
6. For a complete host evaluation, additionally run `explore`, `frame`, `design`, `architecture`, `plan` and `correct-course` on a fresh fictional brief, preserving each output and its one qualified next command.
|
|
27
27
|
|
|
28
28
|
Read the project's CONTRIBUTING and accepted decisions throughout. Evaluate all fourteen stages before marking full native workflow coverage. An absent discovery entry, broken relative link, unexecuted check reported as passed, forbidden write or unexplained permission expansion is a failure, not a cosmetic issue.
|
|
29
|
+
|
|
30
|
+
Version 0.4.1 installs the offline report runtime with scoped-delivery. Tests execute it in all three exported layouts under CommonJS, ESM and package-less projects, intercepting only the OS-opening boundary. These tests validate report generation and dispatch, not native menu discovery or graphical browser rendering. See [release details](docs/RELEASE-0.4.1.md).
|
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ DevMethod exposes fourteen `devmethod-*` workflow commands in the agent’s skil
|
|
|
32
32
|
|
|
33
33
|
A reusable workflow for taking a software project from exploration to delivery: decisions, UX, architecture, tickets, development, tests, review and handoff. Six focused skills support fourteen workflow stages, each ending with evidence, limitations and one suggested next command.
|
|
34
34
|
|
|
35
|
-
**DevMethod 0.4.
|
|
35
|
+
**DevMethod 0.4.1.** [Complete review-to-report flow](docs/RELEASE-0.4.1.md) · [0.3 workflow changes](docs/RELEASE-0.3.0.md). This release completes review generation and opening inside the agent, with an installed offline renderer. Review guidance now follows sensitive-data outputs, failure recovery and affected contracts beyond the diff; [detection evaluation](evaluation/review-detection/README.md) separates reproducible defects from measured reviewer results. Check the registry and GitHub release for publication evidence.
|
|
36
36
|
|
|
37
37
|
For developers and small teams using coding agents in new or existing repositories. Requires Node.js 22+ and npm; Git is required for context provenance. Application examples have separate framework/database prerequisites. DevMethod records scope, decisions and verification; it does not certify agent output, infer all dependencies, deploy applications or run an autonomous backlog. Installation and deterministic fixture results are separate from native host validation. See [compatibility](COMPATIBILITY.md).
|
|
38
38
|
|
|
@@ -43,18 +43,18 @@ Start with [missions and the tested source quick start](docs/MISSIONS.md), the [
|
|
|
43
43
|
Requires Node.js 22+ and npm. Install into a fresh staging directory first:
|
|
44
44
|
|
|
45
45
|
```bash
|
|
46
|
-
npx --yes devmethod-ai@0.4.
|
|
46
|
+
npx --yes devmethod-ai@0.4.1 init --tool codex --dest ../foundation-staging
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
Choose `codex`, `claude` or `cursor`. If you omit `--tool`, an interactive terminal asks. For example:
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
npx --yes devmethod-ai@0.4.
|
|
52
|
+
npx --yes devmethod-ai@0.4.1 init --tool claude --dest ../foundation-staging --dry-run
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
Remove `--dry-run` to write. Select a subset with `--modules decision-architecture,scoped-delivery`; `project-foundation` is always included. Without `--modules`, all six modules are installed. The installer refuses divergent files and duplicate skills across host directories. It never edits AGENTS.md, CLAUDE.md or your package.json. Review the staging output, then merge only what the project needs.
|
|
56
56
|
|
|
57
|
-
The installer has no runtime dependencies and makes no network requests after npm obtains the package. To pin the final version, use `npx --yes devmethod-ai@0.4.
|
|
57
|
+
The installer has no runtime dependencies and makes no network requests after npm obtains the package. To pin the final version, use `npx --yes devmethod-ai@0.4.1 init ...`. To pin a reviewed repository commit instead, use: `npx --yes --package=github:montassarkhalloufi/DevMethod#<commit-sha> devmethod init ...`.
|
|
58
58
|
|
|
59
59
|
Complete PROJECT_PROFILE.md with your real stack, commands, scope, deployment permissions and data requirements. Merge AGENTS.foundation.md into the project's existing instructions only after review. Claude Code reads CLAUDE.md: preserve its current content and, if the project has AGENTS.md, optionally add `@AGENTS.md` to import it. Keep existing accepted architecture decisions authoritative.
|
|
60
60
|
|
|
@@ -135,15 +135,19 @@ See [research, decision dialogue and mission migration](docs/WORKFLOW-0.3.md). S
|
|
|
135
135
|
|
|
136
136
|
[Detailed recorded Lisière chain](docs/media/full-chain-4k/README.md) · [Short Clair demo](docs/media/from-zero/README.md) · [Run Clair](examples/clair-from-zero/README.md). Clair is a separate from-zero example. The featured film retains the Lisière story and adds an explicitly separate fictional review example.
|
|
137
137
|
|
|
138
|
-
##
|
|
138
|
+
## Review your changes and open the report
|
|
139
139
|
|
|
140
|
-
|
|
140
|
+
In your coding agent, ask:
|
|
141
141
|
|
|
142
|
-
```
|
|
143
|
-
|
|
142
|
+
```text
|
|
143
|
+
$devmethod-review the current changes, then open the report
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
-
|
|
146
|
+
In Claude Code or Cursor, use `/devmethod-review` with the same request. The agent inspects the actual changes, runs relevant checks, records evidence-backed findings and justified impact, then generates the Markdown and interactive HTML reports from the real review JSON and opens the HTML. You do not need to launch npx, a terminal command or a server. The renderer is installed with scoped-delivery and runs locally with Node.js 22+.
|
|
147
|
+
|
|
148
|
+
For a small review without a requested report, the result can stay in the conversation. On a headless machine or if browser opening fails, the generated artifacts are preserved and linked with the opening limitation. Existing reports are never overwritten. See [review workflow and report delivery](docs/REVIEWS.md).
|
|
149
|
+
|
|
150
|
+
The screenshots below use clearly fictional data to illustrate the interface; your review uses actual project results. The separate terminal viewer remains available for manual use and demos, as described in the [review guide](docs/REVIEW-GUIDE.md).
|
|
147
151
|
|
|
148
152
|

|
|
149
153
|
|
package/START_HERE.md
CHANGED
|
@@ -24,3 +24,5 @@ Examples:
|
|
|
24
24
|
This kit reduces repetitive framing; it does not by itself prove application quality or production readiness.
|
|
25
25
|
|
|
26
26
|
For a new product, `explore` researches existing solutions and discusses the findings before framing. `architecture` discusses structural options before dependent detail; `plan` discusses delivery scope and stays conditional on open decisions. Keep accepted choices, delegations and approved design. For substantial work without conventions, propose a mission PLAN with tickets and a dated REPRISE; keep existing single-file missions usable.
|
|
27
|
+
|
|
28
|
+
To review and view results in one request, use `$devmethod-review the current changes, then open the report` (Codex) or `/devmethod-review` with the same request (Claude Code/Cursor). The agent performs the review and generates/opens the real report using the renderer installed with scoped-delivery. No user-run npx command is needed.
|
package/dist/doctor.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { tools, modules, templates } from './init.js';
|
|
5
5
|
import { commandSkills } from './commands.js';
|
|
6
|
+
import { isReviewRuntimePath } from './review-runtime.js';
|
|
6
7
|
import { parseJson, checkPath, stat } from './filesystem.js';
|
|
7
8
|
function object(value) {
|
|
8
9
|
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
@@ -29,7 +30,8 @@ export function validateManifest(value) {
|
|
|
29
30
|
const rootFile = templates.includes(name) || name === 'DEVMETHOD-LICENSE';
|
|
30
31
|
const skillFile = parts.slice(0, 2).join('/') === root && supportedSkills.includes(parts[2]) &&
|
|
31
32
|
((parts.length === 4 && parts[3] === 'SKILL.md') ||
|
|
32
|
-
(parts.length >= 5 && ['assets', 'references'].includes(parts[3] ?? '') && name.endsWith('.md'))
|
|
33
|
+
(parts.length >= 5 && ['assets', 'references'].includes(parts[3] ?? '') && name.endsWith('.md')) ||
|
|
34
|
+
isReviewRuntimePath(parts));
|
|
33
35
|
if (!safe || (!rootFile && !skillFile))
|
|
34
36
|
throw new Error(`Manifest contains an unsupported path: ${name}`);
|
|
35
37
|
if (typeof hash !== 'string' || !/^[a-f0-9]{64}$/i.test(hash))
|
package/dist/init.js
CHANGED
|
@@ -3,6 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
5
|
import { commandSkills } from './commands.js';
|
|
6
|
+
import { reviewRuntimeFiles } from './review-runtime.js';
|
|
6
7
|
import { parseJson, checkPath, stat } from './filesystem.js';
|
|
7
8
|
export const tools = { codex: '.agents/skills', claude: '.claude/skills', cursor: '.cursor/skills' };
|
|
8
9
|
export const modules = ['project-foundation', 'decision-architecture', 'design-to-code', 'react-feature-engineering', 'reliable-ai-integration', 'scoped-delivery'];
|
|
@@ -114,6 +115,18 @@ export function bundledFiles(tool, selected) {
|
|
|
114
115
|
files.set(`${tools[tool]}/${name}/${relative}`, profile(fs.readFileSync(path.join(source, relative)), tool));
|
|
115
116
|
}
|
|
116
117
|
}
|
|
118
|
+
if (selected.includes('scoped-delivery')) {
|
|
119
|
+
for (const name of reviewRuntimeFiles) {
|
|
120
|
+
const compiled = name.endsWith('.mjs') ? name.replace(/\.mjs$/, '.js') : name;
|
|
121
|
+
const source = path.join(packageRoot, 'dist', compiled);
|
|
122
|
+
checkPath(source);
|
|
123
|
+
let data = fs.readFileSync(source);
|
|
124
|
+
// Explicit .mjs modules work inside CommonJS, ESM and package-less projects.
|
|
125
|
+
if (name.endsWith('.mjs'))
|
|
126
|
+
data = Buffer.from(data.toString().replace(/(from ['"]\.\/[^'"]+)\.js(['"])/g, '$1.mjs$2'));
|
|
127
|
+
files.set(`${tools[tool]}/scoped-delivery/scripts/${name}`, data);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
117
130
|
for (const template of templates)
|
|
118
131
|
files.set(template, profile(fs.readFileSync(path.join(packageRoot, '.agents/skills/project-foundation/assets', template)), tool));
|
|
119
132
|
files.set('DEVMETHOD-LICENSE', fs.readFileSync(path.join(packageRoot, 'LICENSE')));
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { parseArgs } from 'node:util';
|
|
3
|
+
import { prepareReview } from './review-cli.js';
|
|
4
|
+
import { openReview } from './review-open.js';
|
|
5
|
+
// Installed with scoped-delivery. The agent supplies its real review record;
|
|
6
|
+
// no demo mode, package download or repository command is available here.
|
|
7
|
+
try {
|
|
8
|
+
const { values, positionals } = parseArgs({ options: {
|
|
9
|
+
dest: { type: 'string' }, review: { type: 'string' },
|
|
10
|
+
output: { type: 'string' }, markdown: { type: 'string' }, open: { type: 'boolean' },
|
|
11
|
+
}, allowPositionals: false });
|
|
12
|
+
if (positionals.length || !values.review || !values.output || !values.markdown) {
|
|
13
|
+
throw new Error('Expected --review REAL_JSON --output FRESH_HTML --markdown FRESH_MD [--dest PROJECT] [--open].');
|
|
14
|
+
}
|
|
15
|
+
const result = prepareReview({ destination: values.dest ?? process.cwd(), review: values.review,
|
|
16
|
+
output: values.output, markdown: values.markdown });
|
|
17
|
+
console.log(JSON.stringify(result, null, 2));
|
|
18
|
+
if (values.open)
|
|
19
|
+
openReview(result.outputs[0]);
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
23
|
+
process.exitCode = 2;
|
|
24
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Exact installed runtime allowlist; never permit arbitrary scripts in manifests. */
|
|
2
|
+
export const reviewRuntimeFiles = [
|
|
3
|
+
'review-agent.mjs', 'review-cli.mjs', 'review-open.mjs', 'review-model.mjs',
|
|
4
|
+
'review.mjs', 'records.mjs', 'filesystem.mjs', 'review-browser.js', 'review-ui.css',
|
|
5
|
+
];
|
|
6
|
+
export function isReviewRuntimePath(parts) {
|
|
7
|
+
return parts.length === 5 && parts[2] === 'scoped-delivery' && parts[3] === 'scripts' &&
|
|
8
|
+
reviewRuntimeFiles.some(name => name === parts[4]);
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# ADR 010: Complete review delivery with an installed renderer
|
|
2
|
+
|
|
3
|
+
Status: accepted under the user's request to fix review generation/opening and update GitHub/npm documentation, 2026-09-13.
|
|
4
|
+
|
|
5
|
+
The previous adapter stopped at findings when the separate CLI was unavailable. Install a narrow report entry point and its existing renderer dependencies with scoped-delivery. This directly completes the requested interaction using the accepted offline HTML design. Asking the user to launch npx is no longer the primary workflow. A runtime download on every review would retain a network dependency; duplicating the rendering implementation would create competing behavior. Instead derive installed modules from the same compiled sources and preserve exact browser bytes.
|
|
6
|
+
|
|
7
|
+
Use explicit .mjs module filenames to work regardless of the adopting project's package type. The manifest permits only the enumerated runtime files in scoped-delivery/scripts, never arbitrary scripts. Installation preflight, hashes and update preview apply to these files. The helper accepts a real result JSON and fresh HTML/Markdown paths, optionally opening through the existing fixed OS adapter. It performs no review itself and has no demo mode. The agent owns inspection and evidence before calling it.
|
|
8
|
+
|
|
9
|
+
Keep quick reviews light and respect explicit report-only/headless requests. When opening is requested, execute generation and local opening under that authorization. Preserve files on opening failure and report the precise limit. Revisit only for demonstrated runtime portability problems or an actual need for live reports; no server, unrelated design change or extra command is introduced.
|
package/docs/COMMANDS.md
CHANGED
|
@@ -27,10 +27,10 @@ A full installation exposes all fourteen commands. Foundation-only installs expo
|
|
|
27
27
|
|
|
28
28
|
For example, select `$devmethod-review` and add `the current uncommitted diff against HEAD`. The agent reads actual changes and relevant contracts, executes applicable checks and reports located findings, evidence and limits. A clear ticket, PR, revision or path selection can replace that target. An ambiguous scope is clarified only when it affects the review.
|
|
29
29
|
|
|
30
|
-
A review does not silently fix product code. Small reviews can stay in the conversation; substantial reviews use the existing tracker or the installed structured review format. Checks that were not run remain explicitly unverified.
|
|
30
|
+
A review does not silently fix product code. Small reviews can stay in the conversation; substantial reviews use the existing tracker or the installed structured review format. Checks that were not run remain explicitly unverified. To inspect and view results in one request, use `$devmethod-review the current changes, then open the report`. The agent generates Markdown/HTML from its real JSON and opens the report using the installed offline renderer. It does not ask you to run npx. A small review may stay in the conversation unless a report is requested. The separate shell viewer only presents existing records and remains available for manual use; fictional demo results never replace your review.
|
|
31
31
|
|
|
32
32
|
## Adopt into an existing project
|
|
33
33
|
|
|
34
|
-
Install the candidate into a fresh staging directory with the same host and selected modules. Compare it with your existing installation. Copy the new `devmethod-*` folders and merge the relevant foundation/delivery resources, preserving local customizations. Do not overwrite the filled project profile, instructions, mission records or a divergent skill. Keep the previous manifest until you have intentionally reconciled all adopted baseline files; never replace hashes merely to hide modifications. `update-preview` can classify changes read-only; it does not apply them.
|
|
34
|
+
Install the candidate into a fresh staging directory with the same host and selected modules. Compare it with your existing installation. Copy the new `devmethod-*` folders and merge the relevant foundation/delivery resources, including scoped-delivery/scripts for offline report generation, preserving local customizations. Do not overwrite the filled project profile, instructions, mission records or a divergent skill. Keep the previous manifest until you have intentionally reconciled all adopted baseline files; never replace hashes merely to hide modifications. `update-preview` can classify changes read-only; it does not apply them.
|
|
35
35
|
|
|
36
36
|
Do not run `init` over a customized installation expecting it to upgrade: conflicts block all writes. Both legacy single-file missions and PLAN/tickets missions remain readable, with no automatic migration. If the host does not discover the added skills, reload/reopen its session and inspect its configured skill directory. As a fallback ask it to read the installed `devmethod-review/SKILL.md` directly. Discovery and model behavior need host-specific verification; file installation tests alone do not prove autocomplete behavior.
|
package/docs/EVALUATION.md
CHANGED
|
@@ -37,3 +37,7 @@ Publish a result only with its pinned input, actual evidence and limitations. Pr
|
|
|
37
37
|
See [the native evaluation workspace](../evaluation/README.md) for pinned B1/B3/B4/B5 fixtures, preparation, before/after file evidence, immutable acceptance checks and host evidence requirements. B2 still needs a pinned React project and approved screen. Host observations and raw transcripts remain local until explicitly approved for publication.
|
|
38
38
|
|
|
39
39
|
Matched batch records are checked by [the comparison validator](../evaluation/COMPARISONS.md). This validates consistency and preserves unavailable/failure denominators; it does not execute models or prove comparative outcomes.
|
|
40
|
+
|
|
41
|
+
## Evidence-backed review detection
|
|
42
|
+
|
|
43
|
+
The [review detection cases](../evaluation/review-detection/README.md) add sensitive-data sinks, concurrent/partial/timeout effects, unchanged consumers/old data and a correct control. Reproduce the seeds separately from running reviews, retain original findings before oracle adjudication, and report detected/missed defects and false positives. Fixture tests and report schema checks do not demonstrate model detection; no new model performance claim is made without observed runs.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# DevMethod 0.4.1
|
|
2
|
+
|
|
3
|
+
Fix the incomplete review-to-report flow introduced in 0.4.0. Ask `devmethod-review` to review changes and open the report: the agent performs the review, generates real JSON/Markdown/HTML results and opens the HTML itself. README and review guides now lead with this agent workflow; terminal demos are secondary.
|
|
4
|
+
|
|
5
|
+
The scoped-delivery installation now includes a narrowly allowlisted offline renderer. It needs Node.js 22+ but no npm, network, global CLI or application package dependency. Explicit .mjs modules work in CommonJS, ESM and package-less projects. The existing trusted renderer and OS opening adapter are reused; the installed entry point accepts only an explicit real record and fresh report paths, with no demo mode. Invalid records and existing outputs remain protected. Opening failure retains the generated files.
|
|
6
|
+
|
|
7
|
+
Adopt through a fresh staging installation and intentional merging, including scoped-delivery/scripts and the updated review instructions. Preserve customized files and old manifests; new diagnostics support older installations. The legacy shell viewer remains compatible. Product code and design behavior are unchanged.
|
|
8
|
+
|
|
9
|
+
Automated verification covers all three host layouts and all three package modes, report content from a supplied fixture, schema rejection, overwrite protection, OS-opening dispatch and runtime path allowlisting. OS-opening dispatch is intercepted in tests and is not proof of a graphical browser opening. Full native agent behavior is separate from these runtime checks. Executed counts, CI and npm verification are recorded in the GitHub release.
|
|
10
|
+
|
|
11
|
+
Review instructions now trace relevant sensitive data to logs/errors/API/telemetry, probe failure and recovery scenarios, inspect affected consumers and old data beyond the diff, and require concrete impact for architecture criticism. Application leak detection remains distinct from report redaction. Controls stay proportionate to risk. The existing evaluation protocol gains four review cases with five reproducible seeded defects, a correct control and explicit missed/false-positive accounting. Fixture/scorer validation is not a measured model detection improvement.
|
|
12
|
+
|
|
13
|
+
A [bounded independent review](../evaluation/review-detection/observed-0.4.1/README.md) confirmed 4 of 5 expected defects with no observed false positive. The timeout scenario was mentioned as a risk but not reproduced and remains unconfirmed. One small run without a matched baseline does not establish improved general detection.
|
package/docs/REVIEW-GUIDE.md
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
DevMethod 0.3 adds a review workflow and an offline browser interface. Version 0.3.1 expands the documentation and the existing narrated film with this journey. The CLI presents recorded results; it does not inspect code or run tests on your behalf.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Review and open your real results
|
|
6
|
+
|
|
7
|
+
Use `$devmethod-review the current changes, then open the report` in Codex, or `/devmethod-review` with the same request in Claude Code/Cursor. The agent inspects the work and executes relevant checks, records actual findings and limits, generates JSON/Markdown/HTML, and opens the report. The renderer ships with scoped-delivery: no user-run npx command or server is needed. Browser opening failures retain the files and are reported separately from review findings. See [the complete flow](REVIEWS.md).
|
|
8
|
+
|
|
9
|
+
The following terminal example is only a manual tour of fictional data, not the main workflow.
|
|
6
10
|
|
|
7
11
|
## Try the packaged example
|
|
8
12
|
|
package/docs/REVIEWS.md
CHANGED
|
@@ -2,9 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
The agent's `devmethod-review` command (also `project-foundation review`) performs project-aware inspection under [the review workflow](../.agents/skills/scoped-delivery/references/review-workflow.md). The CLI `devmethod review` only validates and presents recorded results. It never runs repository commands, discovers evidence on disk, or performs the review itself.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## From review to open report
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
After installation, invoke `$devmethod-review the current changes, then open the report` in Codex, or `/devmethod-review` with that request in Claude Code/Cursor. The agent performs the inspection and checks, writes the real result JSON, then generates Markdown and HTML and opens the report. No terminal command is required from the user. The installed scoped-delivery renderer works offline with Node.js 22+ and does not require npm or a global CLI.
|
|
8
|
+
|
|
9
|
+
The agent follows [report delivery](../.agents/skills/scoped-delivery/references/review-report.md). Generation preserves earlier snapshots; failed opening preserves the files and produces an explicit limitation. A report-only request omits browser opening. Small reviews can remain in the conversation unless a report is requested.
|
|
10
|
+
|
|
11
|
+
## Manual viewer and fictional demo (optional)
|
|
8
12
|
|
|
9
13
|
After 0.3.0 publication, from a temporary or project directory:
|
|
10
14
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Review detection cases
|
|
2
|
+
|
|
3
|
+
This extends the existing [workflow evaluation protocol](../../docs/EVALUATION.md) for review quality. It is not another delivery workflow. Four small cases contain five known defects and a correct control. All credentials and personal data in these fixtures are synthetic.
|
|
4
|
+
|
|
5
|
+
## Cases and expected findings
|
|
6
|
+
|
|
7
|
+
The evaluator-owned [oracle](oracle.json) records expected findings and their scope. The [reproduction tests](reproduce.test.mjs) demonstrate application behavior directly, before any report masking:
|
|
8
|
+
|
|
9
|
+
| Case | Expected | Evidence |
|
|
10
|
+
|---|---|---|
|
|
11
|
+
| SENSITIVE | LEAK: token/email and provider error reach logs, telemetry and API output | Captured synthetic outputs contain the sentinel values |
|
|
12
|
+
| SUBMISSION | RACE, PARTIAL, TIMEOUT: duplicate effects under interleaving, interrupted notification, or effect-before-timeout | Controlled gates and injected failures observe two effects for one key |
|
|
13
|
+
| COMPATIBILITY | CONTRACT: changed response breaks existing stored data and an unchanged consumer | Missing total and NaN invoice result |
|
|
14
|
+
| CONTROL | No defect under its stated trim/reject contract | Contract tests pass; adding an architecture layer is a preference, not a blocking defect |
|
|
15
|
+
|
|
16
|
+
Run `node --test evaluation/review-detection/reproduce.test.mjs`. These tests deliberately assert the presence of seeded defects: a green result means the evaluation input is reproducible, not that the application is correct or a model detected anything. They are included in npm test through tests/review-detection.test.mjs.
|
|
17
|
+
|
|
18
|
+
## Conduct a detection evaluation
|
|
19
|
+
|
|
20
|
+
Use fresh isolated projects and the existing protocol's pinned method/host/model, equal budgets, permissions and transcript requirements. Give the reviewer only the case's fixture files, the relevant project contract and this neutral task: “Review these changes and affected interactions. Report evidence-backed defects, impact and unverified risks; do not fix code.” For COMPATIBILITY identify compatibility.mjs as changed and keep consumer.mjs available as unchanged context. For CONTROL preserve the comment's accepted contract. Do not provide oracle.json, reproduction tests, expected IDs or this case table to the reviewer. Keep the oracle outside the review workspace; editing seed files invalidates the run.
|
|
21
|
+
|
|
22
|
+
Save the original findings before consulting the oracle. A separate evaluator compares scenarios, locations, evidence and impact, not keywords. Link each confirmed finding to one or more expected defect IDs only when the reproduction and impact support the match. A risk still awaiting verification is not a confirmed detection. Treat a duplicate description of the same defect as one detected defect. Classify unsupported blockers (including personal architecture/style preferences on the control) as false positives. Do not automatically call a new, unmatched finding false: investigate and mark unresolved until adjudicated; update the oracle only with independent evidence and version the change.
|
|
23
|
+
|
|
24
|
+
Use [scoreDetection](score.mjs) with expected IDs, the original findings' IDs/confidence and separate evidence-based adjudications. Record per-case detected/missed IDs, false positives, unresolved and unadjudicated findings, plus elapsed time and scope. Attach the actual reviewed evidence and impact justification; booleans in a scoring input are attestations, not proof. Report the control separately (zero expected defects means recall is unavailable, not 100%). Preserve failed, interrupted and empty runs in denominators. Test calibration deliberately includes misses and a false positive to check accounting, not to claim model performance.
|
|
25
|
+
|
|
26
|
+
## Evidence and limits for 0.4.1
|
|
27
|
+
|
|
28
|
+
Fixture development and oracle-aware inspection establish five reproducible defects and the bounded control contract. Scorer calibration verifies that three misses and one false positive remain visible in a constructed input; duplicate matches cannot inflate detected counts. This is runtime/oracle verification, not a blind review campaign. A [single independent review run](observed-0.4.1/README.md) confirmed 4/5 expected defects with no observed false positive; the timeout case remained unverified. No improvement over 0.4.0/BMAD or general detection rate is established. Written review guidance, valid JSON, report redaction and report opening prove none of those claims.
|
|
29
|
+
|
|
30
|
+
Coverage is deliberately bounded: synthetic local functions with injected collaborators; no real providers, production telemetry settings, process-kill crash recovery, database isolation or complete contract migration matrix. Add relevant cases when actual changes expose those risks; do not require this entire suite for every small project edit.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Deliberately defective evaluation input. Only synthetic values are used in tests.
|
|
2
|
+
export async function connectAccount(input, provider, outputs) {
|
|
3
|
+
try {
|
|
4
|
+
return await provider(input.token);
|
|
5
|
+
} catch (error) {
|
|
6
|
+
const detail = { token: input.token, email: input.email, message: error.message };
|
|
7
|
+
outputs.log(detail);
|
|
8
|
+
outputs.telemetry({ event: 'connect-failed', detail });
|
|
9
|
+
return { status: 502, body: detail };
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// Deliberately defective evaluation input: completion is recorded after all effects.
|
|
2
|
+
export async function submit(key, state, charge, notify) {
|
|
3
|
+
if (state.has(key)) return state.get(key);
|
|
4
|
+
const receipt = await charge(key);
|
|
5
|
+
await notify(receipt);
|
|
6
|
+
state.set(key, receipt);
|
|
7
|
+
return receipt;
|
|
8
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Observed bounded review, 0.4.1 candidate
|
|
2
|
+
|
|
3
|
+
Date: 2026-09-13. One independent delegated Codex agent reviewed four isolated cases using the candidate installed skills. Its supplied context contained TASK.md, the five source modules and installed method resources, not the oracle, expected defect IDs or author reproduction tests. Deliberately-defective comments were removed from the supplied cases; the control's accepted contract was retained. The exact source bytes are included and hashed. All source hashes were unchanged after review.
|
|
4
|
+
|
|
5
|
+
The evaluator produced [raw findings](raw-findings.md), [executable probes](review-probes.mjs) and [captured results](probe-results.json). Run `node review-probes.mjs` from this directory to reproduce those probes. The root agent subsequently inspected the artifacts against the oracle and recorded [adjudication](adjudication.json); [score.json](score.json) is derived with scoreDetection, not a keyword matcher. The adjudicator authored the oracle; this is disclosed, not an independent human judgment.
|
|
6
|
+
|
|
7
|
+
| Case | Expected confirmed defects | Detected | Missed/unconfirmed | False positives |
|
|
8
|
+
|---|---:|---:|---|---:|
|
|
9
|
+
| Sensitive outputs | 1 | 1 | none | 0 |
|
|
10
|
+
| Submission/recovery | 3 | 2 | TIMEOUT | 0 |
|
|
11
|
+
| Compatibility | 1 | 1 | none | 0 |
|
|
12
|
+
| Correct control | 0 | 0 | not applicable | 0 |
|
|
13
|
+
|
|
14
|
+
Observed: 4/5 expected defects confirmed (80% on this tiny set); no false positive among the four confirmed findings. Charge-success-then-timeout was named as an unknown risk but not exercised, so TIMEOUT remains missed/unconfirmed rather than credited as a confirmed detection. The correct control received no invented architectural blocker. Application sink exposure was tested before any report masking.
|
|
15
|
+
|
|
16
|
+
Limits: one run, five simple known defects, no matched earlier-version or BMAD baseline, no production systems, no human blind adjudication. This demonstrates bounded actual review behavior, not a general detection rate or an improvement over another version/method. Native autocomplete, graphical report opening, process-kill recovery and real provider/database behavior are not tested. The review was explicitly given a skill path; discovery was not measured. The raw narrative/probes are retained, not a complete host transport transcript. Model identifier, metered usage/cost and elapsed time were not independently captured and remain unavailable. Reviewer Node version was v23.10.0.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Review each case independently, without changing code. All files in case-a and case-b are the proposed implementation. In case-c, only compatibility.mjs changed; consumer.mjs is existing context. In case-d, control.mjs is the proposed implementation and its comment states the accepted contract. Report evidence-backed defects, impact and unverified risks. No report opening or product fixes are requested. Use only this directory and its installed skills. Do not inspect other temporary folders or repositories.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"expected": [
|
|
3
|
+
"LEAK",
|
|
4
|
+
"RACE",
|
|
5
|
+
"PARTIAL",
|
|
6
|
+
"TIMEOUT",
|
|
7
|
+
"CONTRACT"
|
|
8
|
+
],
|
|
9
|
+
"findings": [
|
|
10
|
+
{
|
|
11
|
+
"id": "DET-A-001",
|
|
12
|
+
"confidence": "confirmed"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "DET-B-001",
|
|
16
|
+
"confidence": "confirmed"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "DET-B-002",
|
|
20
|
+
"confidence": "confirmed"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"id": "DET-C-001",
|
|
24
|
+
"confidence": "confirmed"
|
|
25
|
+
}
|
|
26
|
+
],
|
|
27
|
+
"adjudications": [
|
|
28
|
+
{
|
|
29
|
+
"findingId": "DET-A-001",
|
|
30
|
+
"verdict": "matched",
|
|
31
|
+
"defectIds": [
|
|
32
|
+
"LEAK"
|
|
33
|
+
],
|
|
34
|
+
"evidenceReviewed": true,
|
|
35
|
+
"impactJustified": true
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"findingId": "DET-B-001",
|
|
39
|
+
"verdict": "matched",
|
|
40
|
+
"defectIds": [
|
|
41
|
+
"RACE"
|
|
42
|
+
],
|
|
43
|
+
"evidenceReviewed": true,
|
|
44
|
+
"impactJustified": true
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"findingId": "DET-B-002",
|
|
48
|
+
"verdict": "matched",
|
|
49
|
+
"defectIds": [
|
|
50
|
+
"PARTIAL"
|
|
51
|
+
],
|
|
52
|
+
"evidenceReviewed": true,
|
|
53
|
+
"impactJustified": true
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"findingId": "DET-C-001",
|
|
57
|
+
"verdict": "matched",
|
|
58
|
+
"defectIds": [
|
|
59
|
+
"CONTRACT"
|
|
60
|
+
],
|
|
61
|
+
"evidenceReviewed": true,
|
|
62
|
+
"impactJustified": true
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export async function connectAccount(input, provider, outputs) {
|
|
2
|
+
try {
|
|
3
|
+
return await provider(input.token);
|
|
4
|
+
} catch (error) {
|
|
5
|
+
const detail = { token: input.token, email: input.email, message: error.message };
|
|
6
|
+
outputs.log(detail);
|
|
7
|
+
outputs.telemetry({ event: 'connect-failed', detail });
|
|
8
|
+
return { status: 502, body: detail };
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"case-d/control.mjs": "4e853b97ef8ca8976417d415c73954cf8d28d287fe933045fb4d4be787d2e607",
|
|
3
|
+
"case-c/compatibility.mjs": "b5ddd4fae3cae50b273c44311a2d238018def2f13019bc46f9098832d48ed08f",
|
|
4
|
+
"case-c/consumer.mjs": "fdd545577dcdd27331ed7cad4705ebdc916b59a429f46f1050e67d6fde4266ed",
|
|
5
|
+
"case-b/submission.mjs": "bae2ae0aee50ae04bd12ba7296b7f9ec86119a48dfd5b2ade1b7aa486ba6eb98",
|
|
6
|
+
"case-a/sensitive.mjs": "436f69d20aff30205560910622d0bfeac66de54aac6f4c2b519ccc69ed4a0420"
|
|
7
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
{
|
|
2
|
+
".agents/skills/devmethod-status/SKILL.md": "ec9aa89614a0604f35b223064b3492d5a659a1726728d2f8eca5969ac984f73c",
|
|
3
|
+
".agents/skills/devmethod-plan/SKILL.md": "52cc2a73da6b8bb210e7f2ee5f3e3965594036d721069a831e28793790ad2812",
|
|
4
|
+
".agents/skills/react-feature-engineering/SKILL.md": "7766b7db86833cc63d16493b1d0b006a9a2c275ac513fcbbae6d3d2e4efafd95",
|
|
5
|
+
".agents/skills/devmethod-architecture/SKILL.md": "db7e80b11863176455034a1ab03dba6fcf4784ea92f0bbeee5ba223fc991189b",
|
|
6
|
+
".agents/skills/devmethod-verify/SKILL.md": "00549c3fb4cfd9b7d98d37128c5c6bb117605c286afdb36df5a03c038f072e30",
|
|
7
|
+
".agents/skills/design-to-code/SKILL.md": "5747b5368cf6d7bdec982b6ea1d2337bf0beb96604866ff043838d6030184c1c",
|
|
8
|
+
".agents/skills/project-foundation/SKILL.md": "e9cff76b8570e22dedc0b43af816d5455ddd3848c459d8190025b6248a01107c",
|
|
9
|
+
".agents/skills/devmethod-integrate/SKILL.md": "f7a83a03b3e34d654341bc1638f3c66bda43732e5a8bebb6639d7df772cf770b",
|
|
10
|
+
".agents/skills/reliable-ai-integration/SKILL.md": "cc43feeb4f4ba892248ce4fe9637c31637b6078eb35de7ebcc08aa918c6ff85f",
|
|
11
|
+
".agents/skills/devmethod-handoff/SKILL.md": "1bb1a8874aae43d5cff3d57863d59d4d67dcf0fd320573aa746fa39bb94d5f02",
|
|
12
|
+
".agents/skills/devmethod-design/SKILL.md": "a9c3fa2168e8962d838d9eba9f6659073249695645e6348c4594818cbfcec7a4",
|
|
13
|
+
".agents/skills/devmethod-next/SKILL.md": "4ad95bb95e88278fd131755d019ee085215a62f592353a6076566cc81b481aeb",
|
|
14
|
+
".agents/skills/scoped-delivery/SKILL.md": "557f9624a735917bfd0eb2971a32496c17fdf9d63f6c361f0d2405aaad297065",
|
|
15
|
+
".agents/skills/devmethod-ready/SKILL.md": "8501aa3a54eaa3ebafa1ae68b0364350cc6cb0082fb23cb6c46b34d4499d4314",
|
|
16
|
+
".agents/skills/devmethod-explore/SKILL.md": "61eefe4ea7ace88e8baa974d9aaa9129fa8ac91b60d9033b3434283bd74b46e5",
|
|
17
|
+
".agents/skills/devmethod-correct-course/SKILL.md": "774b8bc371886f386489c9de398712d4e1d35045d0ad773a3913e4b67f74ac98",
|
|
18
|
+
".agents/skills/devmethod-review/SKILL.md": "ef727903c1be13dbcbfc40ebbf6a862b748f10e13aaa8e8dfe9a5569169b078d",
|
|
19
|
+
".agents/skills/devmethod-implement/SKILL.md": "a5f607c29959891acfb54aa4cb3a19ecf5d58bde10125dce6dbf7f3933a69bba",
|
|
20
|
+
".agents/skills/decision-architecture/SKILL.md": "31c065dd526498e3a42b9182bac38196c25b7b98311808ae6416f98949f84b3d",
|
|
21
|
+
".agents/skills/devmethod-frame/SKILL.md": "371b123419d01de049ff27637f73e4d85ef167847e4d16c6aa47b2d20d4a1a6b",
|
|
22
|
+
".agents/skills/decision-architecture/references/backend-boundaries.md": "df6c66f6a12bb4749c0de5e346b7d8312ad8d5e05298b6e9283fdda6f401dab0",
|
|
23
|
+
".agents/skills/decision-architecture/references/product-decisions.md": "5b656d98684f57adee4980eb6853b1d2a78a7b69943b5e8e14104702adc23be2",
|
|
24
|
+
".agents/skills/decision-architecture/references/api-contracts.md": "085b90efc2c84718db87f4123f68e93b78c40e7fe4fe3d347e5e9840f9a6e5c3",
|
|
25
|
+
".agents/skills/decision-architecture/assets/ADR.md": "4a2525af41cca1189412bd7c36b11a41cc0a5f95beccf9fd49149ab97aa3cc4c",
|
|
26
|
+
".agents/skills/scoped-delivery/references/review-workflow.md": "c049418c2cb4c1e64f462a83471ae5d1a91def2a210aeb78c4d0addd70cacfae",
|
|
27
|
+
".agents/skills/scoped-delivery/references/review-report.md": "d7f0260bf58b7b65e2105031aa90c66e80fd690f7e0ec826b7af87d426690630",
|
|
28
|
+
".agents/skills/scoped-delivery/references/review-format.md": "19ac482c790b752e1bc3a7ca733fa34411a0b359371b7a7d1246af8c27e05f0e",
|
|
29
|
+
".agents/skills/scoped-delivery/references/verification-and-cost.md": "d732e351c8408a4848fa4afa720fc2d261ecea4158e482b6d219c32ede6a3a7f",
|
|
30
|
+
".agents/skills/scoped-delivery/scripts/review-cli.mjs": "b43cebaab81857bab3c538a68581d895a88cf022559999eeebe35a4b296c79b8",
|
|
31
|
+
".agents/skills/scoped-delivery/scripts/review-model.mjs": "9e295ba34f0fd87f13ef40ab96ac784bd3896f74c3d02c5e79e05606089150b0",
|
|
32
|
+
".agents/skills/scoped-delivery/scripts/records.mjs": "0e02f386c3e7fad694326c13a4c17f993c1d1e1f6703fe20c981cabbfce0a3de",
|
|
33
|
+
".agents/skills/scoped-delivery/scripts/review-open.mjs": "aef4e126988b9a1927cd0f52426fee52cdc3878b73f149447ba51430339f19e4",
|
|
34
|
+
".agents/skills/scoped-delivery/scripts/review-agent.mjs": "c8deabf384a2fb0ce30f678ed3ed9a2fc4f3584562a86c752133ff5b8fea9ff2",
|
|
35
|
+
".agents/skills/scoped-delivery/scripts/filesystem.mjs": "c655f3189d63fe09291e3932a4619b314333c72748adc1551867b9112a127643",
|
|
36
|
+
".agents/skills/scoped-delivery/scripts/review.mjs": "6a594e6fdd566b244a2118aa2d5125a805f69774e466f37dda8ea950dd6cc912",
|
|
37
|
+
".agents/skills/scoped-delivery/scripts/review-browser.js": "0c7bb25cb58f14eee941cd0995b0d3a544c66ef55f6ef027e8deb71e76314692",
|
|
38
|
+
".agents/skills/scoped-delivery/scripts/review-ui.css": "d428603fe454dbabb2fd4b8c0d5042f959e56ffc0046bce20c7413acadbf122e",
|
|
39
|
+
".agents/skills/scoped-delivery/assets/MISSION.md": "3f5d16b7763cb9a50ac9c131eae038f1fd386d6dc013247e7e3d7deb3a7246e1",
|
|
40
|
+
".agents/skills/scoped-delivery/assets/CHECKPOINT.md": "f74ebb2181c1944289270e389ea1df7ab7e3feb1aa975d6f89104e82ab73717b",
|
|
41
|
+
".agents/skills/scoped-delivery/assets/VERIFICATION.md": "46569dc339f41be3dfb44ed89f1ec67140b1e491c8ba11c74c5ca6e6b455f6ca",
|
|
42
|
+
".agents/skills/scoped-delivery/assets/SLICE.md": "8bf1c238904da20f5fb09c42f427356988b478b2230a4ad26cff5c6dd00c4ee8",
|
|
43
|
+
".agents/skills/scoped-delivery/assets/REPRISE.md": "8e43cfc004bfd8a341063d02be4d656b9d3ca71abb7082ad75f00c2bf2d4ceb2",
|
|
44
|
+
".agents/skills/scoped-delivery/assets/TICKET.md": "94befa7d148691d6094684715d950abf4f3ba617adeee0b80037ee9fc1cb8f6b",
|
|
45
|
+
".agents/skills/scoped-delivery/assets/PLAN.md": "5f4c7f1f3e2918008da50d1865227e97f4e438633690b8bfe58bbe3e9211fe04",
|
|
46
|
+
".agents/skills/scoped-delivery/assets/REVIEW.md": "414c7f578303048dc60c816190fa88463571a75f5e747d0d2948b830fb3e0df1",
|
|
47
|
+
".agents/skills/reliable-ai-integration/references/jobs-and-costs.md": "32de8493813b84dc4b2540e4cd6a682494e0357dfb4bd94002e36c5e080ad699",
|
|
48
|
+
".agents/skills/reliable-ai-integration/references/evidence-and-media.md": "67100fe297e40729a8acb02840ec1d6f3974e0f082b512cd203b76c4d45dcda0",
|
|
49
|
+
".agents/skills/reliable-ai-integration/assets/AI_EVALUATION.md": "9fccf7fbadb38f10210d6813fde059678646f978e72bf9e79fb5f7b2366910c1",
|
|
50
|
+
".agents/skills/project-foundation/references/mission-context.md": "9738b9959a08e827997a90219fe084be537bb0093acb31e5ab1688ff6e59d75c",
|
|
51
|
+
".agents/skills/project-foundation/references/operating-commands.md": "03631d3adabd62e127e6ac6780e4a94e30442ee4595cb490549ca5cc77557a01",
|
|
52
|
+
".agents/skills/project-foundation/references/work-sizing.md": "d2b10e269610eb6d8adf57426477e540e519cbae2d21a90a649e0abe0179509e",
|
|
53
|
+
".agents/skills/project-foundation/references/delivery-planning.md": "5727abcdaca48a7db6d65d81e55a211df07db92f755f11cd714c92e67273892d",
|
|
54
|
+
".agents/skills/project-foundation/references/exploration.md": "1e6dac589bd16819416d337a3307950215d36ae80155ee7966b57e3cc201b660",
|
|
55
|
+
".agents/skills/project-foundation/assets/ENGINEERING_POLICY.template.md": "341d03b7ef60d6c02badebadc950a535e3db5a82c196b773a523af052db25a98",
|
|
56
|
+
".agents/skills/project-foundation/assets/OPPORTUNITES.md": "ddcb9a24b70d26194795948a95566ddf68b33628ce8b5240b05b5f818d9a644c",
|
|
57
|
+
".agents/skills/project-foundation/assets/CADRAGE.md": "292974445e586b97e4ecd228f4b7147b27da98620c7066af233599b56e60b1cf",
|
|
58
|
+
".agents/skills/project-foundation/assets/EXISTANT.md": "7a20805fb2a0529e858e5f029441898d4c76f9344f8a5a4290c17d9f9dedf011",
|
|
59
|
+
".agents/skills/project-foundation/assets/START_HERE.md": "924a91a603c26eda7126a265d1a3f55f2cca15e5952d0df2af71ed1556d4a0a4",
|
|
60
|
+
".agents/skills/project-foundation/assets/REGLES.md": "cc6ee52e50380ac4c4c9a484b3fccad4f07b968bee073ad3e5f41f2c6180ab2b",
|
|
61
|
+
".agents/skills/project-foundation/assets/PROJECT_PROFILE.md": "80dcc8e1d5d5282411981b0b4b38e5cb258cc39e730a7cf914de8cfd9cb703d0",
|
|
62
|
+
".agents/skills/project-foundation/assets/AGENTS.foundation.md": "9564d30890ce7133e6b1fc9a6115e2217145276d7843069c447ff456d6eea0ea",
|
|
63
|
+
".agents/skills/project-foundation/references/profiles/messaging.md": "56ac919ada8deef6cda92395f48cec50cdd5777b3c5f246356195251141de61b",
|
|
64
|
+
".agents/skills/project-foundation/references/profiles/react-next.md": "8819abb3f7a22fb87e26f6a5465e77aeec5505f5c6dd9462edafbee1918237c9",
|
|
65
|
+
".agents/skills/project-foundation/references/profiles/cloud-delivery.md": "d64452e2e9d1e8e48a9ca4c8c14974583376cf6a1def241e0ad7843eaf02dab7",
|
|
66
|
+
".agents/skills/project-foundation/references/profiles/node-nest.md": "32d3f59086dbc132f1b1c01e87bdd52fa126b01c29a1137ba99ca3a0eb5232f1",
|
|
67
|
+
".agents/skills/project-foundation/references/profiles/README.md": "a05eaaeef6bf25aca4bbd1e65e79995cab8f5419c275cd4d37beeff547be469c",
|
|
68
|
+
".agents/skills/project-foundation/references/profiles/postgres-drizzle.md": "115e801dc009dee06c734167e3c0c3ef1b8f240009c8f2fb3e88656a8116d24a",
|
|
69
|
+
".agents/skills/project-foundation/references/profiles/ci.md": "ca7ab4034c8dae85853069f37d9fc90d7d9a8c5511267df1cf51c9ec035865b9",
|
|
70
|
+
".agents/skills/project-foundation/references/profiles/typescript.md": "c6452bc3f6397eb975267ccb4909e566655513d974e91c6207a7a1b858d41147",
|
|
71
|
+
".agents/skills/project-foundation/references/profiles/mongodb.md": "9ec39ab8209425d714a1d17635aaa21f18995d9fb52123754e5bac87107a9a82",
|
|
72
|
+
".agents/skills/design-to-code/references/visual-creation.md": "cd557e06c3020e24d3e57dc1f862f2a1d5d16c495078fe8b3d8d655e6573806e",
|
|
73
|
+
".agents/skills/design-to-code/references/ux-contract.md": "7769a8830a8b83358d6274b46a0d2f44a54cce7fc3aaef38b9890f2eec0174b1",
|
|
74
|
+
".agents/skills/design-to-code/assets/UI_ACCEPTANCE.md": "fd71bcb4a9f86e36b5bc91ba5a3b4310c7c109cd3bf36cda85b6b0c0e701b817",
|
|
75
|
+
".agents/skills/react-feature-engineering/references/review-and-sources.md": "43a0829aa20a95bb46c09ab5f2ae7d11965abfe54dfaf48fd1e59798f8283ad3"
|
|
76
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"a": {
|
|
3
|
+
"log": {
|
|
4
|
+
"tokenExposed": true,
|
|
5
|
+
"emailExposed": true,
|
|
6
|
+
"providerMessageSecretExposed": true
|
|
7
|
+
},
|
|
8
|
+
"telemetry": {
|
|
9
|
+
"tokenExposed": true,
|
|
10
|
+
"emailExposed": true,
|
|
11
|
+
"providerMessageSecretExposed": true
|
|
12
|
+
},
|
|
13
|
+
"response": {
|
|
14
|
+
"tokenExposed": true,
|
|
15
|
+
"emailExposed": true,
|
|
16
|
+
"providerMessageSecretExposed": true
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"bConcurrent": {
|
|
20
|
+
"charges": 2,
|
|
21
|
+
"notices": 2,
|
|
22
|
+
"receipts": [
|
|
23
|
+
{
|
|
24
|
+
"key": "same-key",
|
|
25
|
+
"sequence": 1
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"key": "same-key",
|
|
29
|
+
"sequence": 2
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"stateSize": 1
|
|
33
|
+
},
|
|
34
|
+
"bRetry": {
|
|
35
|
+
"charges": 2,
|
|
36
|
+
"sizeAfterFailure": 0
|
|
37
|
+
},
|
|
38
|
+
"c": {
|
|
39
|
+
"existingStoredInput": {
|
|
40
|
+
"id": "order-fixture",
|
|
41
|
+
"amountCents": 1250
|
|
42
|
+
},
|
|
43
|
+
"oldResponse": {
|
|
44
|
+
"id": "order-fixture"
|
|
45
|
+
},
|
|
46
|
+
"newResponse": {
|
|
47
|
+
"id": "new-order",
|
|
48
|
+
"totalCents": 1250
|
|
49
|
+
},
|
|
50
|
+
"oldConsumerNaN": true,
|
|
51
|
+
"newConsumerNaN": true,
|
|
52
|
+
"baselineTotal": 12.5
|
|
53
|
+
},
|
|
54
|
+
"d": {
|
|
55
|
+
"stringExamplesPassed": 3,
|
|
56
|
+
"nonStringsRejected": 8
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Independent review findings
|
|
2
|
+
|
|
3
|
+
Reviewed only the supplied case files under `/private/tmp/devmethod-041-detection-run`, following TASK.md and the installed devmethod-review, project-foundation routing/stage contract, scoped-delivery and review-workflow references. Node runtime: v23.10.0. No case files changed. No external calls, browser opening, fixes, or other workflow stages executed. PROJECT_PROFILE.md is an unfilled template and adds no accepted requirements. No manifest, lockfile, CONTRIBUTING.md or AGENTS.md was found in the bounded discovery.
|
|
4
|
+
|
|
5
|
+
## Confirmed defects
|
|
6
|
+
|
|
7
|
+
### DET-A-001 — Credential and personal data escape through the failure detail
|
|
8
|
+
|
|
9
|
+
- Case/location: case-a, `case-a/sensitive.mjs:5-8` (construction at line 5; sinks at lines 6–8).
|
|
10
|
+
- Classification: confirmed defect; severity P1 (credential exposure); confidence high; status open.
|
|
11
|
+
- Trigger: provider rejects after receiving a token, including a rejection whose message contains credential material.
|
|
12
|
+
- Expected/observed: failure reporting should avoid propagating raw credentials and unnecessary personal data. Instead, the raw token, email, and unrestricted provider message are copied into log, telemetry and returned response body.
|
|
13
|
+
- Evidence: `review-probes.mjs` invokes the real function with synthetic sentinel credentials and an independently marked provider error. `probe-results.json:a` records tokenExposed, emailExposed and providerMessageSecretExposed as true for every sink. These booleans are computed against original captured application output before any report sanitization.
|
|
14
|
+
- Concrete impact: any consumer of logs, telemetry or the error response receives the original credential. Provider messages can add further sensitive content, so removing only the explicit token field would leave another exposure path. Actual recipients/retention are not supplied.
|
|
15
|
+
- Proposed correction: construct an allowlisted public error response and separately sanitized diagnostic event; omit token and unnecessary email, and redact or replace untrusted provider messages before sinks. Preserve a safe correlation/error code for diagnosis. Trade-off: less raw diagnostic detail requires deliberate safe diagnostics.
|
|
16
|
+
- Resolution verification: repeat the three-sink sentinel check, covering explicit fields and provider messages; all exposure flags should be false while failure reporting remains usable.
|
|
17
|
+
|
|
18
|
+
### DET-B-001 — Simultaneous submissions pass the same deduplication check
|
|
19
|
+
|
|
20
|
+
- Case/location: case-b, `case-b/submission.mjs:2-5`, especially the awaited charge at line 3 before reservation.
|
|
21
|
+
- Classification: confirmed defect; severity P1 (duplicate charge invocation); confidence high; status open.
|
|
22
|
+
- Trigger: two calls submit the same absent key before either awaited charge completes.
|
|
23
|
+
- Expected/observed: the same-key state check implies duplicate submissions should reuse one result. Both calls observe absence and independently invoke charge and notify; only their final receipts are written to one map slot.
|
|
24
|
+
- Evidence: a deferred Promise in `review-probes.mjs` deliberately holds the first charge, starts a second submit with the same key, then releases both. `probe-results.json:bConcurrent` shows 2 charges, 2 notices, distinct sequence receipts, and stateSize 1.
|
|
25
|
+
- Concrete impact: the function dispatches duplicate charge requests and notifications while its state retains only one receipt. Actual duplicate billing depends on whether an unseen provider independently deduplicates the supplied key.
|
|
26
|
+
- Proposed correction: atomically reserve/share the pending same-key operation before charge. For multiple processes use a durable uniqueness/reservation mechanism, plus provider idempotency with the same operation identity. Define failure recovery so an ambiguous effect is not blindly reissued. Trade-off: reservation lifetime and recovery require explicit handling.
|
|
27
|
+
- Resolution verification: repeat the controlled interleaving and assert exactly one charge and consistent returned receipt for both callers.
|
|
28
|
+
|
|
29
|
+
### DET-B-002 — Notification failure discards knowledge of an already successful charge
|
|
30
|
+
|
|
31
|
+
- Case/location: case-b, `case-b/submission.mjs:4-5`.
|
|
32
|
+
- Classification: confirmed defect; severity P1 (unsafe retry after partial success); confidence high; status open.
|
|
33
|
+
- Trigger: charge succeeds but notify rejects, followed by retry using the same key and state.
|
|
34
|
+
- Expected/observed: the successful financial effect should remain recorded so retry can recover notification without recharging. Because state is updated only after notify, the first call rejects with no saved receipt and the retry invokes charge again.
|
|
35
|
+
- Evidence: `probe-results.json:bRetry` reports sizeAfterFailure 0 and charges 2 after an injected notification exception and one retry.
|
|
36
|
+
- Concrete impact: an ordinary notification outage causes duplicate charge requests and loses the recovery reference to the first successful receipt. Actual duplicate billing remains conditional on provider behavior, but the missing local record and repeated invocation are reproduced.
|
|
37
|
+
- Proposed correction: durably record charged status/receipt at the successful charge boundary, track notification status independently and retry that step. Combine with DET-B-001's reservation and provider reconciliation for charge timeouts. Simply caching before notify must not silently suppress the outstanding notification forever. Trade-off: additional state/status handling.
|
|
38
|
+
- Resolution verification: inject notify failure after a successful charge; retry should preserve the first receipt, charge exactly once, and finish notification recovery.
|
|
39
|
+
|
|
40
|
+
### DET-C-001 — Response rename breaks the existing consumer and persisted order shape
|
|
41
|
+
|
|
42
|
+
- Case/location: case-c, changed `case-c/compatibility.mjs:2`; existing context `case-c/consumer.mjs:1-2`.
|
|
43
|
+
- Classification: confirmed compatibility defect; severity P1 (existing monetary calculation unavailable); confidence high; status open.
|
|
44
|
+
- Trigger: pass existingOrder through orderResponse then invoiceTotal; also reproduced with a new totalCents-shaped order.
|
|
45
|
+
- Expected/observed: existing consumer expects amountCents and the supplied existing record stores amountCents. The changed response reads and exposes only totalCents. Existing records lose the amount entirely (undefined; omitted by JSON serialization), and the consumer returns NaN for both old and new response shapes.
|
|
46
|
+
- Evidence: `probe-results.json:c` shows baselineTotal 12.5 for the unchanged consumer and fixture; oldResponse contains only id when serialized; newResponse contains totalCents 1250; oldConsumerNaN and newConsumerNaN are both true.
|
|
47
|
+
- Concrete impact: existing invoice totals become NaN rather than 12.50, and serialized responses for stored records no longer carry the monetary value. Only compatibility.mjs is proposed change, so a presumed coordinated consumer/data migration cannot justify the break.
|
|
48
|
+
- Proposed correction: preserve amountCents for existing consumers and read existing stored amountCents (optionally add totalCents and a carefully defined fallback if new records must be supported). Alternatively perform an explicitly coordinated/versioned migration of data and consumers, outside the supplied change. Trade-off: a compatibility alias temporarily maintains two names.
|
|
49
|
+
- Resolution verification: test unchanged invoiceTotal(orderResponse(existingOrder)) equals 12.5; include new-format records only if that additional format is an accepted requirement.
|
|
50
|
+
|
|
51
|
+
## Case-d: no confirmed defect
|
|
52
|
+
|
|
53
|
+
`case-d/control.mjs:1` expressly requires trimming strings and rejecting non-strings, with no layering mandate. Lines 2–4 satisfy that contract. Executed checks cover surrounding whitespace, whitespace-only input, already trimmed text and eight non-string values (including a boxed String); all passed. An extra architectural layer or rejecting an empty trimmed result would introduce a preference/new requirement, not correct an observed defect.
|
|
54
|
+
|
|
55
|
+
## Risks and limits, separate from confirmed defects
|
|
56
|
+
|
|
57
|
+
- Case-a: nested SDK error fields other than message are not serialized by this supplied function and are not a separate demonstrated leak. Real provider errors, actual sink redaction, access controls and retention were not available. Verify integration sinks and representative safe synthetic SDK errors to assess the full deployed exposure; no claim of actual credential compromise is made.
|
|
58
|
+
- Case-b: provider-side idempotency, cross-process state consistency, restart durability, and charge-success-then-timeout ambiguity are unknown. The supplied key is passed to charge and might enable downstream deduplication, but no such implementation is supplied. Verify with an idempotency-aware provider stub/integration and fault injection before asserting actual duplicate billing. Notification retries might also duplicate delivered notifications if failure occurs after delivery; not exercised here.
|
|
59
|
+
- Case-c: no migration policy or alternative consumer exists in the supplied scope. Findings rely on the explicitly supplied existing consumer/record, not inferred external clients.
|
|
60
|
+
- Case-d: no length, emptiness, Unicode normalization or layering requirements are given; none are imposed.
|
|
61
|
+
- No external documentation consulted because the assignment forbids external calls. These conclusions rely on small local ECMAScript modules and executed behavior, not version-dependent third-party APIs. No UI exists in scope. No repository suite or CI gate is provided. This is bounded evidence, not production verification.
|
|
62
|
+
|
|
63
|
+
## Executed checks
|
|
64
|
+
|
|
65
|
+
1. Read TASK.md, all five source modules with numbered lines, relevant installed review references, and local profile; bounded file discovery for rules/manifests/lockfiles. Manual source inspection covers every supplied case source line and the case-c interaction.
|
|
66
|
+
2. Read runtime with `node --version`: v23.10.0.
|
|
67
|
+
3. Ran `node review-probes.mjs > probe-results.json`: exit 0. Assertions confirm reproduced defects, so exit 0 means the reproductions behaved as documented, not that case-a/b/c passed correctness checks.
|
|
68
|
+
4. Case-a: three captured failure sinks, direct credential/email and provider-message sentinel checks; confirmed exposure in all sinks.
|
|
69
|
+
5. Case-b: deterministic simultaneous same-key submission and a separate successful-charge/failed-notification/retry sequence; both demonstrate repeated charge calls.
|
|
70
|
+
6. Case-c: unchanged consumer baseline and old/new stored-shape response paths; baseline 12.5, both transformed paths NaN.
|
|
71
|
+
7. Case-d: 3 string examples and 8 non-string rejection checks passed.
|
|
72
|
+
|
|
73
|
+
Artifacts: `review-probes.mjs` (synthetic executable reproduction), `probe-results.json` (captured results), and this requested `raw-findings.md`. No report UI was generated/opened. All four cases were evaluated independently; no external answer material was inspected.
|
|
74
|
+
|
|
75
|
+
Recommended next command (not executed): `devmethod-implement` for the confirmed corrections, after authorizing that scope.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { connectAccount } from './case-a/sensitive.mjs';
|
|
3
|
+
import { submit } from './case-b/submission.mjs';
|
|
4
|
+
import { orderResponse } from './case-c/compatibility.mjs';
|
|
5
|
+
import { invoiceTotal, existingOrder } from './case-c/consumer.mjs';
|
|
6
|
+
import { normalizeLabel } from './case-d/control.mjs';
|
|
7
|
+
const evidence = {};
|
|
8
|
+
const token = 'SYNTHETIC_TOKEN_REVIEW_041';
|
|
9
|
+
const email = 'synthetic@example.invalid';
|
|
10
|
+
const nested = 'SYNTHETIC_ERROR_SECRET_041';
|
|
11
|
+
let log, telemetry;
|
|
12
|
+
const result = await connectAccount({ token, email }, async () => { throw new Error(`Provider rejected credential ${nested}`); }, { log: x => { log = x; }, telemetry: x => { telemetry = x; } });
|
|
13
|
+
evidence.a = Object.fromEntries(Object.entries({log,telemetry,response:result}).map(([sink,value]) => [sink, { tokenExposed: JSON.stringify(value).includes(token), emailExposed: JSON.stringify(value).includes(email), providerMessageSecretExposed: JSON.stringify(value).includes(nested) }]));
|
|
14
|
+
for (const sink of Object.values(evidence.a)) assert.deepEqual(sink,{tokenExposed:true,emailExposed:true,providerMessageSecretExposed:true});
|
|
15
|
+
let release;
|
|
16
|
+
const barrier = new Promise(resolve => { release = resolve; });
|
|
17
|
+
let charges = 0, notices = 0;
|
|
18
|
+
const state = new Map();
|
|
19
|
+
const charge = async key => { const sequence = ++charges; await barrier; return { key, sequence }; };
|
|
20
|
+
const first = submit('same-key',state,charge,async()=>{notices++;});
|
|
21
|
+
const second = submit('same-key',state,charge,async()=>{notices++;});
|
|
22
|
+
release();
|
|
23
|
+
const receipts = await Promise.all([first,second]);
|
|
24
|
+
evidence.bConcurrent = {charges,notices,receipts,stateSize:state.size};
|
|
25
|
+
assert.equal(charges,2);
|
|
26
|
+
let retriesCharged = 0;
|
|
27
|
+
const retryState = new Map();
|
|
28
|
+
try { await submit('retry-key',retryState,async()=>({sequence:++retriesCharged}),async()=>{throw new Error('notification unavailable');}); } catch {}
|
|
29
|
+
const sizeAfterFailure = retryState.size;
|
|
30
|
+
await submit('retry-key',retryState,async()=>({sequence:++retriesCharged}),async()=>{});
|
|
31
|
+
evidence.bRetry = {charges:retriesCharged,sizeAfterFailure};
|
|
32
|
+
assert.equal(retriesCharged,2); assert.equal(sizeAfterFailure,0);
|
|
33
|
+
const oldResponse = orderResponse(existingOrder);
|
|
34
|
+
const newResponse = orderResponse({id:'new-order',totalCents:1250});
|
|
35
|
+
evidence.c = {existingStoredInput:existingOrder,oldResponse,newResponse,oldConsumerNaN:Number.isNaN(invoiceTotal(oldResponse)),newConsumerNaN:Number.isNaN(invoiceTotal(newResponse)),baselineTotal:invoiceTotal(existingOrder)};
|
|
36
|
+
assert.equal(invoiceTotal(existingOrder),12.5);
|
|
37
|
+
assert.ok(Number.isNaN(invoiceTotal(oldResponse)));assert.ok(Number.isNaN(invoiceTotal(newResponse)));
|
|
38
|
+
assert.equal(normalizeLabel(' \t hello \n '),'hello');assert.equal(normalizeLabel(' '),'');assert.equal(normalizeLabel('already trimmed'),'already trimmed');
|
|
39
|
+
for (const value of [null,undefined,1,true,{},[],new String('text'),Symbol('label')]) assert.throws(()=>normalizeLabel(value),TypeError);
|
|
40
|
+
evidence.d = {stringExamplesPassed:3,nonStringsRejected:8};
|
|
41
|
+
console.log(JSON.stringify(evidence,null,2));
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"expected": 5,
|
|
3
|
+
"detected": [
|
|
4
|
+
"LEAK",
|
|
5
|
+
"RACE",
|
|
6
|
+
"PARTIAL",
|
|
7
|
+
"CONTRACT"
|
|
8
|
+
],
|
|
9
|
+
"missed": [
|
|
10
|
+
"TIMEOUT"
|
|
11
|
+
],
|
|
12
|
+
"falsePositives": [],
|
|
13
|
+
"unresolved": [],
|
|
14
|
+
"unadjudicated": [],
|
|
15
|
+
"recall": 0.8,
|
|
16
|
+
"limitation": "Separate evidence-based adjudication required; fixture/score tests alone do not measure model detection."
|
|
17
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"format": 1,
|
|
3
|
+
"cases": [
|
|
4
|
+
{"id":"SENSITIVE","files":["sensitive.mjs"],"defects":[{"id":"LEAK","expected":"Synthetic token and email reach log, telemetry and API response on provider failure; error text is forwarded too."}]},
|
|
5
|
+
{"id":"SUBMISSION","files":["submission.mjs"],"defects":[{"id":"RACE","expected":"Two concurrent calls with the same key can both charge before state is recorded."},{"id":"PARTIAL","expected":"Notification failure after a successful charge leaves no receipt in state; retry charges again."},{"id":"TIMEOUT","expected":"A charge that applies its effect then times out leaves no receipt; retry can duplicate the external effect."}]},
|
|
6
|
+
{"id":"COMPATIBILITY","files":["compatibility.mjs","consumer.mjs"],"defects":[{"id":"CONTRACT","expected":"Renamed field loses existing amountCents data and breaks the unchanged invoice consumer."}]},
|
|
7
|
+
{"id":"CONTROL","files":["control.mjs"],"defects":[]}
|
|
8
|
+
]
|
|
9
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { connectAccount } from './fixtures/sensitive.mjs';
|
|
4
|
+
import { submit } from './fixtures/submission.mjs';
|
|
5
|
+
import { orderResponse } from './fixtures/compatibility.mjs';
|
|
6
|
+
import { invoiceTotal, existingOrder } from './fixtures/consumer.mjs';
|
|
7
|
+
import { normalizeLabel } from './fixtures/control.mjs';
|
|
8
|
+
|
|
9
|
+
test('LEAK: application outputs expose synthetic values before any report masking', async () => {
|
|
10
|
+
const token = 'SYNTHETIC-ONLY-TOKEN', email = 'fixture@example.invalid';
|
|
11
|
+
const logs = [], telemetry = [];
|
|
12
|
+
const result = await connectAccount({ token, email }, async () => { throw new Error(`Provider rejected ${token}`); }, {
|
|
13
|
+
log: x => logs.push(x), telemetry: x => telemetry.push(x),
|
|
14
|
+
});
|
|
15
|
+
for (const sink of [logs, telemetry, result.body]) {
|
|
16
|
+
assert.ok(JSON.stringify(sink).includes(token));
|
|
17
|
+
assert.ok(JSON.stringify(sink).includes(email));
|
|
18
|
+
}
|
|
19
|
+
assert.ok(result.body.message.includes(token));
|
|
20
|
+
});
|
|
21
|
+
test('RACE: controlled double submission duplicates the external effect', async () => {
|
|
22
|
+
let release; const gate = new Promise(resolve => { release = resolve; });
|
|
23
|
+
let charges = 0; const state = new Map();
|
|
24
|
+
const charge = async () => { charges++; await gate; return 'receipt'; };
|
|
25
|
+
const first = submit('same', state, charge, async () => {});
|
|
26
|
+
const second = submit('same', state, charge, async () => {});
|
|
27
|
+
release(); await Promise.all([first, second]);
|
|
28
|
+
assert.equal(charges, 2);
|
|
29
|
+
});
|
|
30
|
+
test('PARTIAL: retry after notification failure duplicates a successful charge', async () => {
|
|
31
|
+
const state = new Map(); let charges = 0;
|
|
32
|
+
const charge = async () => `receipt-${++charges}`;
|
|
33
|
+
await assert.rejects(submit('same', state, charge, async () => { throw new Error('interrupted'); }));
|
|
34
|
+
assert.equal(state.has('same'), false);
|
|
35
|
+
await submit('same', state, charge, async () => {});
|
|
36
|
+
assert.equal(charges, 2);
|
|
37
|
+
});
|
|
38
|
+
test('TIMEOUT: applied external effect can survive a timeout and be retried', async () => {
|
|
39
|
+
const state = new Map(); let effects = 0;
|
|
40
|
+
const charge = async () => { effects++; if (effects === 1) throw new Error('timeout after commit'); return 'receipt'; };
|
|
41
|
+
await assert.rejects(submit('same', state, charge, async () => {}));
|
|
42
|
+
await submit('same', state, charge, async () => {});
|
|
43
|
+
assert.equal(effects, 2);
|
|
44
|
+
});
|
|
45
|
+
test('CONTRACT: existing data and unchanged consumer break outside the diff', () => {
|
|
46
|
+
const response = orderResponse(existingOrder);
|
|
47
|
+
assert.equal(response.totalCents, undefined);
|
|
48
|
+
assert.ok(Number.isNaN(invoiceTotal(response)));
|
|
49
|
+
});
|
|
50
|
+
test('CONTROL: accepted simple implementation meets its contract without extra layers', () => {
|
|
51
|
+
assert.equal(normalizeLabel(' ready '), 'ready');
|
|
52
|
+
assert.throws(() => normalizeLabel(null), TypeError);
|
|
53
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Score separately adjudicated findings, never infer a match from keywords or JSON validity. */
|
|
2
|
+
export function scoreDetection(expectedIds, findings, adjudications) {
|
|
3
|
+
const expected = new Set(expectedIds);
|
|
4
|
+
const ids = new Set(findings.map(f => f.id));
|
|
5
|
+
if (expected.size !== expectedIds.length || ids.size !== findings.length) throw new Error('Duplicate IDs');
|
|
6
|
+
const judged = new Set(), detected = new Set(), falsePositives = [], unresolved = [];
|
|
7
|
+
for (const item of adjudications) {
|
|
8
|
+
if (!ids.has(item.findingId) || judged.has(item.findingId)) throw new Error('Unknown or duplicate finding adjudication');
|
|
9
|
+
judged.add(item.findingId);
|
|
10
|
+
if (item.verdict === 'matched') {
|
|
11
|
+
if (!Array.isArray(item.defectIds) || !item.defectIds.length || item.defectIds.some(id => !expected.has(id))) throw new Error('Unknown or empty defect match');
|
|
12
|
+
const finding = findings.find(f => f.id === item.findingId);
|
|
13
|
+
if (finding.confidence !== 'confirmed') throw new Error('An unconfirmed risk is not a confirmed detection');
|
|
14
|
+
if (!item.evidenceReviewed || !item.impactJustified) throw new Error('Match requires reviewed evidence and justified impact');
|
|
15
|
+
item.defectIds.forEach(id => detected.add(id));
|
|
16
|
+
} else if (item.verdict === 'false-positive') falsePositives.push(item.findingId);
|
|
17
|
+
else if (item.verdict === 'unresolved') unresolved.push(item.findingId);
|
|
18
|
+
else throw new Error('Unknown adjudication verdict');
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
expected: expected.size, detected: [...detected], missed: expectedIds.filter(id => !detected.has(id)),
|
|
22
|
+
falsePositives, unresolved, unadjudicated: findings.filter(f => !judged.has(f.id)).map(f => f.id),
|
|
23
|
+
recall: expected.size ? detected.size / expected.size : null,
|
|
24
|
+
limitation: 'Separate evidence-based adjudication required; fixture/score tests alone do not measure model detection.',
|
|
25
|
+
};
|
|
26
|
+
}
|
package/package.json
CHANGED
|
@@ -15,7 +15,7 @@ try {
|
|
|
15
15
|
run('tar', ['-xzf', '-'], root, 0, fs.readFileSync(path.resolve(archive)));
|
|
16
16
|
const pkg = path.join(root, 'package'); const cli = path.join(pkg, 'dist/cli.js');
|
|
17
17
|
const call = (args, expected = 0) => run(process.execPath, [cli, ...args], root, expected);
|
|
18
|
-
assert.equal(JSON.parse(fs.readFileSync(path.join(pkg, 'package.json'))).version, '0.4.
|
|
18
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(pkg, 'package.json'))).version, '0.4.1');
|
|
19
19
|
run(process.execPath, ['scripts/check-docs.mjs'], pkg);
|
|
20
20
|
assert.match(call(['--help']), /Markdown PLAN\/tickets and legacy missions/);
|
|
21
21
|
for (const resource of ['project-foundation/references/exploration.md', 'project-foundation/references/delivery-planning.md', 'project-foundation/assets/EXISTANT.md', 'project-foundation/assets/OPPORTUNITES.md', 'project-foundation/assets/CADRAGE.md', 'project-foundation/assets/REGLES.md', 'scoped-delivery/assets/PLAN.md', 'scoped-delivery/assets/TICKET.md', 'scoped-delivery/assets/REPRISE.md', 'scoped-delivery/assets/MISSION.md', 'scoped-delivery/assets/REVIEW.md', 'scoped-delivery/references/review-workflow.md']) {
|
|
@@ -43,6 +43,11 @@ try {
|
|
|
43
43
|
assert.ok(entries.includes('devmethod-review'));
|
|
44
44
|
for (const name of entries) assert.ok(fs.statSync(path.join(project, skillRoot, name, 'SKILL.md')).size > 0);
|
|
45
45
|
assert.ok(fs.statSync(path.join(project, skillRoot, 'scoped-delivery/references/review-format.md')).size > 0);
|
|
46
|
+
fs.copyFileSync(path.join(pkg, 'examples/review/review.json'), path.join(project, 'review-fixture.json'));
|
|
47
|
+
const reportRun = JSON.parse(run(process.execPath, [path.join(project, skillRoot, 'scoped-delivery/scripts/review-agent.mjs'), '--dest', project, '--review', 'review-fixture.json', '--output', 'actual-report.html', '--markdown', 'actual-report.md']));
|
|
48
|
+
assert.equal(reportRun.status, 'corrections');
|
|
49
|
+
assert.ok(fs.readFileSync(path.join(project, 'actual-report.html'), 'utf8').includes('Content-Security-Policy'));
|
|
50
|
+
assert.ok(fs.statSync(path.join(project, 'actual-report.md')).size > 0);
|
|
46
51
|
const profile = path.join(project, 'PROJECT_PROFILE.md'); fs.appendFileSync(profile, '\nFictional local customization.\n');
|
|
47
52
|
const before = fs.readFileSync(profile);
|
|
48
53
|
const preview = JSON.parse(call(['update-preview', '--dest', project, '--json']));
|
|
@@ -81,5 +86,5 @@ try {
|
|
|
81
86
|
assert.deepEqual([fs.readFileSync(customized), fs.readFileSync(profile)], before);
|
|
82
87
|
console.log(`Actual legacy tarball: ${upstreamChanged ? 'local/upstream conflict' : 'unchanged upstream with local customization'} detected; filled profile/custom skill preserved.`);
|
|
83
88
|
}
|
|
84
|
-
console.log('Packed 0.4.
|
|
89
|
+
console.log('Packed 0.4.1: three host installs, subset, customization preservation, mission/context/staleness/planning and documentation links passed. No native host execution.');
|
|
85
90
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|