@oooooooooo/ch0de 0.0.0-stage → 1.9.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API-TESTING.md +76 -0
- package/DEBTAUR-INTEGRATION.md +71 -0
- package/EDITING.md +80 -0
- package/EXPERT-PANEL.md +16 -0
- package/INTEGRATIONS.md +75 -0
- package/LICENSE +21 -0
- package/MODELS-DEV-LICENSE +21 -0
- package/PROVIDER-CONNECTIONS.md +48 -0
- package/PROVIDERS.md +17 -0
- package/README.md +523 -2
- package/RELIABILITY.md +95 -0
- package/UX-1.3.md +83 -0
- package/UX-1.4.1.md +13 -0
- package/UX-1.4.md +48 -0
- package/UX-1.5.md +13 -0
- package/UX-1.6.md +7 -0
- package/UX-1.7.3.md +13 -0
- package/UX-1.7.4.md +13 -0
- package/UX-1.7.md +19 -0
- package/acceptance-check.js +27 -0
- package/agent-budget.js +4 -0
- package/api-testing.js +140 -0
- package/automatic-re.js +19 -0
- package/benchmarks/public-baselines.json +29 -0
- package/benchmarks/render-1.4.0.json +8 -0
- package/benchmarks/targets.js +11 -0
- package/browser-check.js +1062 -0
- package/build-progress.js +10 -0
- package/build-quality.js +49 -0
- package/chode.js +7937 -0
- package/debtaur-engine.js +305 -0
- package/debtaur-orchestration.js +456 -0
- package/debtaur-worker.cjs +76 -0
- package/doctor.js +24 -0
- package/edit-transactions.js +79 -0
- package/elapsed.js +28 -0
- package/evidence-ledger.js +26 -0
- package/expert-panel.cjs +42 -0
- package/helios-ledger.js +405 -0
- package/install.cjs +25 -0
- package/integrations.js +30 -0
- package/json-output.js +8 -0
- package/local-memory.js +613 -0
- package/local-models.js +52 -0
- package/mcp-client.js +50 -0
- package/memory.js +511 -0
- package/model-catalog.js +46 -0
- package/native-tools.js +27 -0
- package/nosignups.js +325 -0
- package/observation-progress.js +33 -0
- package/orchestrator.js +299 -0
- package/package.json +160 -6
- package/plan-z6.js +490 -0
- package/project-memory.js +27 -0
- package/provider-capacity.js +13 -0
- package/provider-catalog.js +246 -0
- package/provider-connect.js +66 -0
- package/provider-credentials.js +5 -0
- package/provider-directory.js +8 -0
- package/provider-directory.json +32300 -0
- package/provider-errors.js +11 -0
- package/provider-picker.js +188 -0
- package/provider-policy.js +4 -0
- package/provider-race.js +25 -0
- package/provider-request.js +45 -0
- package/provider-store.js +13 -0
- package/rea-environment.js +5 -0
- package/read-window.js +21 -0
- package/recovery-context.js +3 -0
- package/reliability-memory.js +38 -0
- package/reliability-services.js +34 -0
- package/reliability-store.js +16 -0
- package/request-deadline.js +4 -0
- package/serve-frontend.html +172 -0
- package/settings-reset.js +11 -0
- package/skill-catalog.js +16 -0
- package/task-group.js +22 -0
- package/terminal-graphics.js +39 -0
- package/terminal-logo.js +76 -0
- package/terminal-ui.js +79 -0
- package/tool-call-validation.js +26 -0
- package/tool-health.js +22 -0
- package/tool-history.js +5 -0
- package/toolbox.js +156 -0
- package/tui-provider-picker.js +134 -0
- package/tui-worker.js +32 -0
- package/web-context.js +232 -0
- package/wolfram-math.js +6 -0
- package/worker-transport.js +11 -0
- package/workspace-commands.js +45 -0
- package/workspace-log.js +56 -0
- package/workspace-store.js +7 -0
- package/workspace-tui.js +113 -0
- package/workspace-view.js +109 -0
- package/worktrees.js +18 -0
- package/z6-integration.js +227 -0
package/UX-1.3.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Chode 1.3 — app building, research, and the terminal workspace
|
|
2
|
+
|
|
3
|
+
## Start
|
|
4
|
+
|
|
5
|
+
Run `node chode.js` in an interactive terminal. `chode tui` explicitly opens the
|
|
6
|
+
workspace. The ordinary command line and piped REPL remain available. Set
|
|
7
|
+
`CHODE_LEGACY_REPL=1` for the older interactive readline interface.
|
|
8
|
+
|
|
9
|
+
The workspace keeps the composer pinned while the conversation scrolls. It
|
|
10
|
+
uses a separate engine process, so synchronous tool commands do not freeze
|
|
11
|
+
typing. Ctrl+O opens activity; PgUp/PgDn scroll; Alt+Enter or Ctrl+J adds a line;
|
|
12
|
+
Ctrl+C stops an active run, clears idle input, or exits an empty idle composer.
|
|
13
|
+
Windows multiline input packets stay in the composer even when the terminal
|
|
14
|
+
strips bracketed-paste markers. Paste requires a separate Enter to send.
|
|
15
|
+
|
|
16
|
+
Commands: `/help`, `/details on|off`, `/session name`, `/new`, `/model id|auto`,
|
|
17
|
+
and `/quit`. Responses streamed before final verification are labelled Draft.
|
|
18
|
+
The final result replaces the draft. Activity uses the wide right panel or stays
|
|
19
|
+
available behind Ctrl+O in a wide terminal. The terminal buffer, cursor, raw mode,
|
|
20
|
+
and wrapping are restored on exit. Narrow/wide frames and grapheme widths are
|
|
21
|
+
covered by layout tests.
|
|
22
|
+
|
|
23
|
+
## Apps and evidence
|
|
24
|
+
|
|
25
|
+
Mentioning an HTML file together with JavaScript, CSS, JSON, React or a backend
|
|
26
|
+
now selects the normal agent workflow. Previously this could select single-file
|
|
27
|
+
HTML mode, which disabled the writers needed for a multi-file app. Explicit
|
|
28
|
+
`--output page.html` still selects a single self-contained HTML artifact.
|
|
29
|
+
|
|
30
|
+
Changed JS/CJS/MJS files receive `node --check`; changed JSON must parse. The
|
|
31
|
+
independent verifier receives bounded artifact content as well as observed tool
|
|
32
|
+
results. A syntax error blocks verification. This is stronger evidence than file
|
|
33
|
+
existence, but syntax and a model verdict are not substitutes for browser or
|
|
34
|
+
behavioral tests. JSX/TypeScript still require the project's build tools.
|
|
35
|
+
|
|
36
|
+
Verifier selection prefers keyed providers only when they actually have a key.
|
|
37
|
+
With no configured keys it selects a different keyless provider instead of
|
|
38
|
+
guaranteeing an authentication failure in the verifier lane.
|
|
39
|
+
|
|
40
|
+
Successful reads and page retrievals produce source records with an id, locator,
|
|
41
|
+
retrieval time and SHA-256 content digest. Repeated observations of identical
|
|
42
|
+
content deduplicate; changed content remains a separate observation. Failed
|
|
43
|
+
fetches are not evidence. URL citations that were not retrieved in the run are
|
|
44
|
+
flagged for review. This checks provenance, not whether a source entails a claim.
|
|
45
|
+
Initial automatic page inspection also contributes source records.
|
|
46
|
+
|
|
47
|
+
`chode ai "prompt" --json` produces a versioned result envelope with completion
|
|
48
|
+
status, answer, sources and usage. Queued/partial/unverified JSON results exit
|
|
49
|
+
unsuccessfully. HTML artifacts are labelled ARTIFACT_SAVED, not VERIFIED.
|
|
50
|
+
|
|
51
|
+
## Public comparison targets
|
|
52
|
+
|
|
53
|
+
`npm run benchmark:targets` prints the dated public baseline snapshot in
|
|
54
|
+
`benchmarks/public-baselines.json`. No competitor API calls are made.
|
|
55
|
+
|
|
56
|
+
- Terminal-Bench 4.0: Claude Code / Opus 5.5 max — 64.8% ± 3.1 percentage points.
|
|
57
|
+
Source: https://www.tbench.ai/ (hydrated 4.0 leaderboard, retrieved 2026-10-17).
|
|
58
|
+
- Terminal-Bench 2.0, Opus 4.5: Claude Code 52.1% ± 2.5; OpenCode 51.7%, with
|
|
59
|
+
no confidence interval supplied. Source: https://www.tbench.ai/?version=2.0 .
|
|
60
|
+
These older rows are not comparable to 4.0 and do not establish a significant
|
|
61
|
+
difference between the two harnesses.
|
|
62
|
+
- WebDev Arena, 2026-10-16: Claude Opus 5.5 Max — 1807 ± 14 Arena score.
|
|
63
|
+
Source: https://arena.ai/leaderboard/code/webdev . This is a model/build
|
|
64
|
+
preference score, not a Claude Code harness score or a percentage.
|
|
65
|
+
- Research methodology: DeepResearch Bench II evaluates recall, analysis, and
|
|
66
|
+
presentation across 132 tasks / 9,430 rubrics. Its project page marks the older
|
|
67
|
+
score table as superseded and notes a judge change, so that table is not used
|
|
68
|
+
as a current numeric target. Source:
|
|
69
|
+
https://agentresearchlab.com/benchmarks/deepresearch-bench-ii/ .
|
|
70
|
+
|
|
71
|
+
Chode has no measured score on those public benchmarks. No public number is
|
|
72
|
+
assigned to Chode, and no superiority claim is made. The local regression fixtures
|
|
73
|
+
check product behavior; they are not a substitute for those benchmark protocols.
|
|
74
|
+
No local competitor runs are part of this release's validation.
|
|
75
|
+
|
|
76
|
+
## Validation
|
|
77
|
+
|
|
78
|
+
The default suite now also runs the classify/plan/verify/repair, guard, provider,
|
|
79
|
+
budget, edit, self-state, and core-tool fixtures. New fixtures cover multi-file app
|
|
80
|
+
routing, malformed JS/JSON rejection, source records, JSON output, worker IPC,
|
|
81
|
+
layout bounds, Unicode, multiline paste and cancellation. Real Windows PTY smoke
|
|
82
|
+
checks cover launch, help, paste and exit. Originals are backed up before applying
|
|
83
|
+
the local release; no website deployment is included.
|
package/UX-1.4.1.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Chode 1.4.1 — native analysis reliability
|
|
2
|
+
|
|
3
|
+
- Added the missing `chode tool rea doctor` and `chode tool rea analyze <local-file> [--timeout <ms>]` CLI routes with JSON output and failure exit codes.
|
|
4
|
+
- JSON error responses, nonzero exits and empty evidence no longer count as successful binary analysis. Unrecognized doctor results report unknown readiness.
|
|
5
|
+
- Added all existing REA regression tests to the default suite, plus false-success regression cases.
|
|
6
|
+
|
|
7
|
+
Live validation used a locally compiled, harmless Windows x64 PE fixture. The actual Chode -> pinned REA 6.5.0 -> Ghidra 12.1.4 path returned SHA-256 219e0b4e1853e6c22a3664809e860da04d4f5663cd43909caea835915ea5c92f, 19 segments, 149 procedures and 279 strings. These are static inventory observations, not a claim that the business logic was recovered. Binary contents were not sent to a model provider.
|
|
8
|
+
|
|
9
|
+
The complex multi-file HTTP application run is recorded separately from passing local regressions. Provider availability is not a passing build, and no general superiority is claimed.
|
|
10
|
+
|
|
11
|
+
Complex-build outcome: the default agent run reached its 240-second external test timeout and created no application files. Separate bounded Kilo and OVH attempts returned HTTP 503 and a request timeout. This remains an unresolved provider-dependent quality gate. No successful complex build is claimed.
|
|
12
|
+
|
|
13
|
+
Run the opt-in native integration gate with an existing Ghidra/Java process environment and `CHODE_REA_TARGET` pointing to a local test executable: `npm run test:rea:live`. The test verifies SHA-256, nonempty inventory and unchanged executable bytes. It never executes the target.
|
package/UX-1.4.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Chode 1.4 — dogfooded workflows
|
|
2
|
+
|
|
3
|
+
This release was exercised using Chode's actual tool dispatcher and exact-match
|
|
4
|
+
edit tool. A live keyless model run built a 349-byte reverse-words program; the
|
|
5
|
+
dogfood fixture independently executed it and checked its output against a fixed
|
|
6
|
+
sample. Local workflow checks cover tools, commands, queues, sessions, jobs,
|
|
7
|
+
rewind, UI editing and recovery. Auth/provisioning and network integrations are
|
|
8
|
+
covered by guarded/mock fixtures; this release did not create accounts or spend
|
|
9
|
+
paid API credits to exercise them.
|
|
10
|
+
|
|
11
|
+
## What improved
|
|
12
|
+
|
|
13
|
+
- The terminal now exposes /tools, /models, /status, /doctor, /todo, /sessions,
|
|
14
|
+
/queue, /retry, /jobs, /rewind, /sources, /activity, /plan and /task. Slash
|
|
15
|
+
suggestions appear while typing. Invalid provider ids are rejected locally.
|
|
16
|
+
- Conversations are saved locally and restored on launch and session switching.
|
|
17
|
+
Transcript storage is bounded; the pending queue keeps the complete failed
|
|
18
|
+
prompt rather than truncating it to 500 characters.
|
|
19
|
+
- Failed interactive requests now enter the queue with their originating session.
|
|
20
|
+
Failed retries retain their task; successful retries acknowledge it. Queue
|
|
21
|
+
writes are atomic; older tasks are no longer silently dropped after 20 entries.
|
|
22
|
+
- Todo and pending-task ids include random suffixes to avoid same-millisecond
|
|
23
|
+
collisions.
|
|
24
|
+
- Multiline up/down/home/end editing works. Recalling history preserves the
|
|
25
|
+
unsent draft. SIGINT/SIGTERM restore the terminal, and stopping work preserves
|
|
26
|
+
the visible conversation.
|
|
27
|
+
- Activity start/completion events compact into one row. Diagnostic notes remain
|
|
28
|
+
available through /activity even when the side panel was closed.
|
|
29
|
+
- Read-only plan mode cannot escape its restrictions through the task tool.
|
|
30
|
+
Uncaught one-shot command errors now exit unsuccessfully.
|
|
31
|
+
- `chode doctor` checks runtime completeness, state JSON and common configuration
|
|
32
|
+
issues. `chode doctor --fix` can convert a UTF-16 .npmrc to UTF-8 while preserving
|
|
33
|
+
its contents and keeping a private local backup. It never prints credentials.
|
|
34
|
+
- The landing page reflects the release, includes the previously omitted Kilo
|
|
35
|
+
provider, and removes the unsupported infinite-uptime claim.
|
|
36
|
+
|
|
37
|
+
## Measured rendering improvement
|
|
38
|
+
|
|
39
|
+
The fixed local repaint scenario uses 40 existing messages, 30 composer updates,
|
|
40
|
+
and a 120x32 terminal. Version 1.3 took 162.85 ms; this candidate took 2.58 ms in
|
|
41
|
+
that run, about 63x faster. This is a warmed terminal-rendering microbenchmark,
|
|
42
|
+
not an end-to-end model speedup or a general superiority claim. The result is
|
|
43
|
+
recorded in benchmarks/render-1.4.0.json. Bounded line-fitting and message-wrap
|
|
44
|
+
caches eliminate repeated work; resizing and changed content still invalidate
|
|
45
|
+
the relevant cached output.
|
|
46
|
+
|
|
47
|
+
Public competitor scores remain separate. Chode has no published score on those
|
|
48
|
+
benchmarks, and this release does not claim benchmark supremacy.
|
package/UX-1.5.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Chode 1.5 — automatic investigation and bounded recovery
|
|
2
|
+
|
|
3
|
+
Chode now automatically inspects a local native binary when the task calls for understanding it and names one existing target (.exe, .dll, .elf, .so or .dylib). Quote paths containing spaces. Ordinary source work, remote URLs, missing paths and ambiguous multiple targets do not trigger native analysis. `--no-re` or `--no-tools` disables this preflight. Analysis is capped at 60 seconds, does not execute the target, and supplies only bounded observed metadata to model reasoning. Missing tooling is reported as a limitation. Metadata is not recovered program behavior; further decompilation still depends on supported REA capabilities.
|
|
4
|
+
|
|
5
|
+
- Workspace requests have a three-minute active-time limit. `/timeout <10–1800 seconds>` changes the limit for subsequent requests. Waiting for an answer to a question pauses the clock. On timeout Chode stops the worker tree and preserves the complete prompt and originating session for `/retry`.
|
|
6
|
+
- Corrupt or malformed pending-task files are never silently treated as empty. Chode reports the problem and preserves the original file, preventing later writes from erasing the queue.
|
|
7
|
+
- `chode doctor --rea` and `/doctor --rea` inspect the Ghidra/Java process settings and offer a concrete diagnostic command. Doctor also validates state shape and checks the new runtime modules.
|
|
8
|
+
- Starting a new conversation saves the previous transcript and clears stale source references. Queue acknowledgement errors no longer crash the workspace after receiving an answer. Informational activity entries are no longer shown as successes.
|
|
9
|
+
- Failed one-shot prompts retain their originating session in the queue.
|
|
10
|
+
|
|
11
|
+
Regression gates cover actual cmdAI automatic-RE integration with isolated model/native subprocess fixtures, explicit target routing, opt-out, bounded metadata, queue corruption, a genuinely hung worker, question-time exclusion and cancellation. Live native integration remains an opt-in gate. No benchmark superiority or successful complex model build is claimed by these reliability improvements.
|
|
12
|
+
|
|
13
|
+
On Windows, REA can reuse existing user/machine Ghidra and Java paths when the launching process has stale settings. Only these two paths are read, existing process settings take precedence, and no registry or global environment changes are made. No analysis engines are installed automatically.
|
package/UX-1.6.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Chode 1.6 — easier terminal workflow
|
|
2
|
+
|
|
3
|
+
The workspace has a centered start screen, a bordered composer, a quieter header, dark/light/system themes, and Ctrl+P command access. Tab cycles matching slash commands; Shift+Tab reverses. `/connect` explains provider setup without inviting credentials in prompts. `/export` saves the actual conversation as private Markdown in `.chode/exports`.
|
|
4
|
+
|
|
5
|
+
The gallery contains four assisted builds: a unit converter, CSV expense summary, architecture studio site and habit tracker. Live keyless requests produced drafts for the converter, expense tool and studio site; several requests timed out. The drafts failed independent quality checks and needed assisted repairs through Chode's local tools. The habit tracker was authored through those tools after its model request failed. These results are not evidence of fully autonomous build reliability.
|
|
6
|
+
|
|
7
|
+
Validation includes boundary cases for both programs, browser interactions for the sites, narrow/wide Unicode frame checks, command cycling, themes, exported content, and the full regression suite. OpenCode was used as an interaction reference, not as a measured parity claim.
|
package/UX-1.7.3.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Website completion in 1.7.3
|
|
2
|
+
|
|
3
|
+
Full and multi-page websites use the agent workspace rather than silently becoming one HTML page. Requests with a backend or several HTML deliverables cannot use the single-file --output mode; omit that option and name the required files in the brief. --check works with website agent builds.
|
|
4
|
+
|
|
5
|
+
Each tool continuation retains the original request, tool instructions, completion contract and file progress. Planning, independent verification and repair retain substantially more of the original brief. The verifier must assess every requested requirement, beyond the model's own plan.
|
|
6
|
+
|
|
7
|
+
For website agent builds, verification checks requested HTML deliverables, local navigation/assets, fragment targets, unfinished placeholders and inline classic JavaScript syntax. When package.json defines a production build, that build must pass. Failures feed the existing bounded repair loop.
|
|
8
|
+
|
|
9
|
+
Single-file website artifacts receive the same static integrity checks and an independent model review against their full brief. Missing requirements allow up to two repairs within the existing shared model budget. Empty or rejected repairs cannot publish the previous invalid draft. Existing files remain untouched when generation fails.
|
|
10
|
+
|
|
11
|
+
Static checks and independent model review do not prove visual quality or live integration behavior. Browser interaction and viewport checks still require available browser tooling, and unavailable credentials/services must be reported explicitly. There is no claim that every arbitrary prompt can be fulfilled flawlessly.
|
|
12
|
+
|
|
13
|
+
Validation: full offline regression suite; controlled model fixtures with real filesystem/tool execution show incomplete three-page builds repaired, semantic omissions rejected/repaired, failed production builds rejected, and empty repairs leaving no artifact. Real free-provider attempts were blocked by OVH timeouts and LLM7 HTTP 429; they did not establish live website completion.
|
package/UX-1.7.4.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Free-provider resilience in 1.7.4
|
|
2
|
+
|
|
3
|
+
Use `chode ai "Build the complete website ..." --free` for strict keyless/local provider routing. With no API keys configured, the TUI starts in free mode automatically. Otherwise use `/free on`; `/free off` restores normal selection. Free mode excludes API-key providers, including accounts advertised as free tiers that might incur usage charges. It also applies to planning, isolated tasks, repair and independent verification.
|
|
4
|
+
|
|
5
|
+
Provider HTTP 429 and 503 cooldowns honor Retry-After seconds or HTTP dates. Missing rate-limit headers use exponential cooldowns; transient connection failures use a short cooldown. Cooldowns survive worker restarts, expire, and are cleared by successful calls. The router can use another available provider during the cooldown. It does not reset quotas, create accounts, rotate identities or bypass provider limits.
|
|
6
|
+
|
|
7
|
+
Agent build progress is saved after planning and each tool result. Retrying the same prompt and session restores the plan and observed file changes, asks the model to inspect current files, and reruns verification. Verified completion clears the checkpoint. `--fresh` ignores the previous build checkpoint without deleting existing project files. Acceptance-test digests distinguish checkpoints with different protected requirements.
|
|
8
|
+
|
|
9
|
+
Free builds use the agent workspace by default, allowing multi-file increments instead of an oversized HTML-only response. Standard provider output-limit metadata survives buffered/streaming routing. Incomplete tool JSON is not executed; the model gets up to two requests to use smaller complete chunks. Partial work remains partial.
|
|
10
|
+
|
|
11
|
+
With free mode enabled and the TUI open, unfinished work can resume automatically after a provider cooldown or a bounded work window. There are at most three automatic retries. The wait is visible and Ctrl+C cancels it; new prompts or session/provider changes cancel the old wait. Long cooldowns remain queued. Closing the UI stops automatic work; saved files, prompts and build checkpoints remain on disk. This is not an always-on background service.
|
|
12
|
+
|
|
13
|
+
Validation includes real tool/filesystem execution with controlled provider responses, persistent cooldown reloads, free-only failover, no paid provider calls, incomplete-output recovery, checkpoint resume through verification, and real child-worker TUI retry/cancellation/budget scenarios. Live free-provider testing during this work encountered OVH timeouts and LLM7 HTTP 429; no claim of guaranteed availability or universal completion is made.
|
package/UX-1.7.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Chode 1.7 — bounded autonomous repair
|
|
2
|
+
|
|
3
|
+
The agent loop shares a budget of at most 30 model-stage calls across planning, execution, verification and nested repairs. Provider attempts default to a 30-second timeout with at most two configured retries per stage. This is a bounded request loop, not a perpetual background agent. Existing permission checks and the workspace request deadline remain in place.
|
|
4
|
+
|
|
5
|
+
Use `chode ai "Implement the calculator" --check tests/acceptance.cjs` to supply an existing JavaScript acceptance test inside the workspace. The loop runs that test, includes the actual failure diagnostics in repair prompts, repairs automatically for at most two rounds, and reruns the checks. The original test digest is checked before and after execution; changing it prevents verification. JavaScript syntax checks now cover every recorded changed file, not only the first twelve. Model verification remains a separate gate: passing a model-written self-test alone does not establish correctness.
|
|
6
|
+
|
|
7
|
+
Verification uses the provider identity actually returned, and cannot pass if that identity matches the drafter or is absent. Repair tool activity and questions remain visible in the UI. Repairs with no observable changes stop early instead of spending another identical round.
|
|
8
|
+
|
|
9
|
+
Failed, unverified, pending and partial workspace runs stay queued with the original prompt/session and bounded failure details. `/retry` carries those diagnostics forward as untrusted context, with an instruction to recheck current files and preserve completed work. Successful verified retries can acknowledge the queued item.
|
|
10
|
+
|
|
11
|
+
Offline end-to-end validation wrote an incorrect calculator, ran a real failing acceptance process, repaired it through the actual edit tool, reran acceptance successfully, and required a separate verifier identity. Model responses were deterministic fixtures; no live-model success rate is claimed. Additional regressions cover provider fallback, shared call limits, changed tests, no-progress repair and recovery of all unfinished statuses.
|
|
12
|
+
|
|
13
|
+
In the workspace, use `/check tests/acceptance.cjs` to select a test for subsequent builds, `/check` to inspect the selection, and `/check off` to clear it. Failed tasks preserve the selected check when queued so `/retry` can use it after reopening the workspace.
|
|
14
|
+
|
|
15
|
+
Release retest: rebuilt and executed a harmless Windows native fixture through Chode (expected result: invoice total 31), reran the real REA CLI hash/immutability gate, and reran automatic RE through actual Ghidra with an isolated model fixture. The 21 existing program checks and responsive website interactions were rerun as well.
|
|
16
|
+
|
|
17
|
+
The live automatic-RE retest exposed filename contamination in task classification: `re-test.exe` was mistaken for a request to run tests. Classification now excludes filenames from verb detection, handles trailing sentence punctuation, and keeps multi-verb read-only requests out of mutation planning/verification. Explicit regression cases cover these routes. This retest validates native inventory and integration, not recovery of arbitrary program behavior from a binary.
|
|
18
|
+
|
|
19
|
+
Acceptance-test selection and its original hash also survive provider failure and workspace timeout. Retry diagnostics retain detailed assertion output while final failure messages stay concise.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const fs=require('fs'),path=require('path'),crypto=require('crypto'),{spawnSync}=require('child_process');
|
|
3
|
+
function locate(root,name){const full=fs.realpathSync(path.resolve(root,name));const relative=path.relative(fs.realpathSync(root),full);if(relative.startsWith('..'+path.sep)||relative==='..'||path.isAbsolute(relative)||!fs.statSync(full).isFile()||! /\.(?:js|cjs|mjs|py)$/i.test(full))throw Error('Acceptance check must be a local JavaScript or Python unittest file inside this workspace.');return full;}
|
|
4
|
+
function digest(full){return crypto.createHash('sha256').update(fs.readFileSync(full)).digest('hex');}
|
|
5
|
+
function capture(root,report){let at=fs.realpathSync(root);for(const part of ['.chode','verification']){at=path.join(at,part);try{if(fs.lstatSync(at).isSymbolicLink())throw Error('Linked verification-report directory');}catch(e){if(e.code!=='ENOENT')throw e;}}fs.mkdirSync(at,{recursive:true});const name='check_'+Date.now()+'_'+crypto.randomBytes(8).toString('hex')+'.json',full=path.join(at,name),temp=full+'.tmp';try{fs.writeFileSync(temp,JSON.stringify(report,null,2),{mode:0o600,flag:'wx'});fs.renameSync(temp,full);}finally{try{fs.unlinkSync(temp);}catch{}}try{const names=fs.readdirSync(at).filter(n=>/^check_\d+_[a-f0-9]{16}\.json$/.test(n)).sort().reverse();for(const old of names.slice(50))fs.unlinkSync(path.join(at,old));}catch{}return path.relative(root,full).replace(/\\/g,'/');}
|
|
6
|
+
function summarizeFailures(text){
|
|
7
|
+
const headings=[...String(text).matchAll(/^(FAIL|ERROR): ([^\r\n]+)/gm)],lines=[];
|
|
8
|
+
for(let i=0;i<Math.min(headings.length,20);i++){
|
|
9
|
+
const h=headings[i],part=String(text).slice(h.index,headings[i+1]?.index),body=part.split(/\r?\n/);
|
|
10
|
+
const error=body.filter(line=>/^[A-Za-z_][\w.]*:/.test(line)&&!/^(?:FAIL|ERROR):/.test(line)).pop()||'';
|
|
11
|
+
lines.push(h[1]+': '+h[2].slice(0,140)+(error?' — '+error.slice(0,190):''));
|
|
12
|
+
}
|
|
13
|
+
if(headings.length>20)lines.push((headings.length-20)+' additional failures; see the full verification report.');
|
|
14
|
+
return lines.join('\n').slice(0,6000);
|
|
15
|
+
}
|
|
16
|
+
function run(root,options={}){
|
|
17
|
+
if(!options.acceptanceTest)return null;
|
|
18
|
+
try{const file=locate(root,options.acceptanceTest),before=digest(file),expected=options.acceptanceDigest||before;if(before!==expected)return {ok:false,detail:'Acceptance test changed during the build; restore the original test.'};const python=/\.py$/i.test(file),command=python?(process.env.CHODE_PYTHON||(process.platform==='win32'?'python':'python3')):process.execPath,args=python?['-c',"import faulthandler, unittest\nfaulthandler.dump_traceback_later(5, repeat=False)\ntry:\n unittest.main(module=None)\nfinally:\n faulthandler.cancel_dump_traceback_later()\n",'discover','-s',path.dirname(file),'-p',path.basename(file)]:[file];const started=Date.now(),r=spawnSync(command,args,{cwd:root,encoding:'utf8',timeout:30000,maxBuffer:1024*1024,windowsHide:true});
|
|
19
|
+
let after;try{after=digest(file);}catch{after=null;}const unchanged=after===expected,pythonTestsRun=python?Number(/Ran (\d+) tests?/.exec(String(r.stdout||'')+String(r.stderr||''))?.[1]||0):null,ok=r.status===0&&!r.error&&!r.signal&&unchanged&&(!python||pythonTestsRun>0);
|
|
20
|
+
const redact=text=>Buffer.from(require('./worker-transport').redact(text,root)).subarray(-16000).toString('utf8').replace(/^\uFFFD/,''),stdout=redact(String(r.stdout||'')),stderr=redact(String(r.stderr||''));
|
|
21
|
+
const failureSummary=python?summarizeFailures(require('./worker-transport').redact(String(r.stdout||'')+'\n'+String(r.stderr||''),root,2*1024*1024)):'';
|
|
22
|
+
const report={schema:1,failureSummary,pythonTestsRun,hangDiagnosticAfterMs:python?5000:null,test:path.relative(root,file).replace(/\\/g,'/'),at:new Date().toISOString(),durationMs:Date.now()-started,expectedDigest:expected,observedDigest:after,unchanged,ok,exitCode:r.status,signal:r.signal||null,error:r.error?String(r.error.message):null,stdout,stderr,stdoutTruncated:Buffer.byteLength(String(r.stdout||''))>16000,stderrTruncated:Buffer.byteLength(String(r.stderr||''))>16000};const reportPath=capture(root,report);
|
|
23
|
+
const detail=!unchanged?'Acceptance test changed while running.':ok?'Independent acceptance test passed':python&&(r.status===0||r.status===5)&&pythonTestsRun===0?'No Python unittest tests were discovered.':((failureSummary?'[failing tests]\n'+failureSummary+'\n':'')+'[stdout]\n'+stdout.slice(-1000)+'\n[stderr]\n'+stderr.slice(-2000)+(r.error?'\n'+r.error.message:''));
|
|
24
|
+
return {ok,detail:detail+'\nVerification report: '+reportPath,reportPath,exitCode:r.status,signal:r.signal||null,durationMs:report.durationMs};
|
|
25
|
+
}catch(error){return {ok:false,detail:error.message};}
|
|
26
|
+
}
|
|
27
|
+
module.exports={locate,digest,run,summarizeFailures};
|
package/agent-budget.js
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
function wrap(aiCall,budget){if(aiCall._chodeBudget===budget)return aiCall;const bounded=async(...args)=>{if(budget.calls>=budget.limit){budget.exhausted=true;return null;}budget.calls++;return aiCall(...args);};bounded._chodeBudget=budget;return bounded;}
|
|
3
|
+
function create(limit){const n=Number(limit);return {calls:0,limit:Number.isInteger(n)&&n>0?Math.min(n,1000):30,exhausted:false};}
|
|
4
|
+
module.exports={wrap,create};
|
package/api-testing.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Optional native test engines. No model requests and no shell-built test commands.
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const crypto = require('crypto');
|
|
6
|
+
const {spawn} = require('child_process');
|
|
7
|
+
const {StringDecoder}=require('string_decoder');
|
|
8
|
+
const activeProcesses=new Set();
|
|
9
|
+
function abortProcesses(signal){for(const stop of [...activeProcesses])stop();process.removeListener('SIGTERM',onTerm);process.removeListener('SIGINT',onInt);if(!process.listenerCount(signal))process.kill(process.pid,signal);}
|
|
10
|
+
function onTerm(){abortProcesses('SIGTERM');}function onInt(){abortProcesses('SIGINT');}
|
|
11
|
+
const engines = {
|
|
12
|
+
hurl: {version:'8.1.0', purpose:'HTTP journeys, captures and assertions', package:'@orangeopensource/hurl@8.1.0', bin:'node_modules/@orangeopensource/hurl/hurl.js', source:'https://hurl.dev/docs/installation.html'},
|
|
13
|
+
schemathesis: {version:'4.32.7', purpose:'Generated OpenAPI/GraphQL contract checks', source:'https://schemathesis.readthedocs.io/en/stable/'},
|
|
14
|
+
k6: {version:'2.3.0', purpose:'Explicit load tests and performance thresholds', source:'https://grafana.com/docs/k6/latest/'}
|
|
15
|
+
};
|
|
16
|
+
const k6Assets = {
|
|
17
|
+
'win32-x64':['windows-amd64.zip','112276d495e5741c968e2bc09ea6196099c1275bd6db9ee0875d173c7148ce43'],
|
|
18
|
+
'linux-x64':['linux-amd64.tar.gz','39c3117b6af817592dcd0ce4242105c0a7af10948c2a425306f0be8f7a8a8ab1'],
|
|
19
|
+
'linux-arm64':['linux-arm64.tar.gz','5ca3433e8201da72a284aaa241a1bb5fb47f4abb4e384d39410ddd8062f49b90'],
|
|
20
|
+
'darwin-x64':['macos-amd64.zip','c83bb16f54f0676afa4ea235fd757f3892d307dbd1bd201450de83af31986654'],
|
|
21
|
+
'darwin-arm64':['macos-arm64.zip','b2417a3038edc5fe81dc178a889237724b595c5c9cfed875822008e46e862c7d']
|
|
22
|
+
};
|
|
23
|
+
function inside(root, file) {const rel=path.relative(root,file);return rel!== '..'&&!rel.startsWith('..'+path.sep)&&!path.isAbsolute(rel);}
|
|
24
|
+
function guardedDir(root, file, create=true) {
|
|
25
|
+
root=path.resolve(root);file=path.resolve(file);
|
|
26
|
+
if(!inside(root,file))throw Error('Directory is outside its configured root');
|
|
27
|
+
const parts=path.relative(root,file).split(path.sep).filter(Boolean);let cur=root;
|
|
28
|
+
for(const part of ['',...parts]){if(part)cur=path.join(cur,part);if(fs.existsSync(cur)){const s=fs.lstatSync(cur);if(s.isSymbolicLink()||!s.isDirectory())throw Error('Test directory must not be a link or file');}else if(create)fs.mkdirSync(cur,{mode:0o700});}
|
|
29
|
+
return file;
|
|
30
|
+
}
|
|
31
|
+
function home(root,create=false) {const base=process.env.CHODE_API_TOOLS_HOME?path.resolve(process.env.CHODE_API_TOOLS_HOME):path.resolve(root);return guardedDir(base,process.env.CHODE_API_TOOLS_HOME?base:path.join(base,'.chode','api-tools'),create);}
|
|
32
|
+
function suite(root,file,engine) {
|
|
33
|
+
if(typeof file!=='string'||!file.trim())throw Error('A workspace test file is required');
|
|
34
|
+
const base=fs.realpathSync(root),full=path.resolve(root,file);
|
|
35
|
+
if(!inside(path.resolve(root),full)||!inside(base,fs.realpathSync(full)))throw Error('Test file must stay inside the workspace');
|
|
36
|
+
const rel=path.relative(root,full).replace(/\\/g,'/');
|
|
37
|
+
if(/(^|\/)(?:\.chode|\.git|\.env[^/]*|credentials?|secrets?)(\/|$)/i.test(rel))throw Error('Credential and internal-state paths cannot be test inputs');
|
|
38
|
+
const s=fs.lstatSync(full);if(!s.isFile()||s.isSymbolicLink()||s.nlink>1||s.size>5*1024*1024)throw Error('Test input must be a regular file under 5 MB');
|
|
39
|
+
const valid=engine==='hurl'?/\.hurl$/i.test(file):engine==='k6'?/\.(?:js|mjs)$/i.test(file):/\.(?:json|yaml|yml|graphql)$/i.test(file);
|
|
40
|
+
if(!valid)throw Error('Unexpected test file extension for '+engine);
|
|
41
|
+
return {full,relative:rel,digest:crypto.createHash('sha256').update(fs.readFileSync(full)).digest('hex')};
|
|
42
|
+
}
|
|
43
|
+
function findProgram(name) {
|
|
44
|
+
for(const dir of String(process.env.PATH||'').split(path.delimiter))for(const suffix of process.platform==='win32'?['.exe','']:['']){const full=path.resolve(dir,name+suffix);try{if(fs.statSync(full).isFile())return full;}catch{}}
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
function npmCLI() {
|
|
48
|
+
const list=[process.env.npm_execpath,path.join(path.dirname(process.execPath),'node_modules/npm/bin/npm-cli.js'),path.join(path.dirname(process.execPath),'../lib/node_modules/npm/bin/npm-cli.js')];
|
|
49
|
+
for(const dir of String(process.env.PATH||'').split(path.delimiter)){list.push(path.join(dir,'node_modules/npm/bin/npm-cli.js'));try{list.push(fs.realpathSync(path.join(dir,'npm')));}catch{}}
|
|
50
|
+
const found=list.find(f=>f&&/npm-cli\.js$/i.test(f)&&fs.existsSync(f));if(!found)throw Error('npm is required for Hurl setup');return found;
|
|
51
|
+
}
|
|
52
|
+
function execute(command,args,options={}) {
|
|
53
|
+
return new Promise(resolve=>{
|
|
54
|
+
const start=Date.now();let stdout='',stderr='',timedOut=false,spawnError=null,finished=false,child;
|
|
55
|
+
const outDecoder=new StringDecoder('utf8'),errDecoder=new StringDecoder('utf8'),cap=128*1024,append=(old,data)=>(old+data).slice(-cap);
|
|
56
|
+
function stop(){if(!child?.pid)return;if(process.platform==='win32'){const killer=spawn('taskkill',['/PID',String(child.pid),'/T','/F'],{windowsHide:true,stdio:'ignore'});killer.on('error',()=>child.kill());}else{try{process.kill(-child.pid,'SIGKILL');}catch{child.kill('SIGKILL');}}}
|
|
57
|
+
function abort(){stop();options.onAbort?.();}
|
|
58
|
+
function done(code,signal){if(finished)return;finished=true;clearTimeout(timer);activeProcesses.delete(abort);if(!activeProcesses.size){process.removeListener('SIGTERM',onTerm);process.removeListener('SIGINT',onInt);}stdout=append(stdout,outDecoder.end());stderr=append(stderr,errDecoder.end());resolve({exitCode:code,signal:signal||null,timedOut,error:spawnError,stdout,stderr,durationMs:Date.now()-start});}
|
|
59
|
+
try{child=spawn(command,args,{cwd:options.cwd,env:{...process.env,NO_COLOR:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8',K6_NO_USAGE_REPORT:'true',...options.env},shell:false,windowsHide:true,detached:process.platform!=='win32',stdio:['ignore','pipe','pipe']});}catch(error){return resolve({exitCode:null,error:error.message,stdout:'',stderr:'',durationMs:Date.now()-start});}
|
|
60
|
+
const timer=setTimeout(()=>{timedOut=true;stop();},options.timeoutMs||60000);
|
|
61
|
+
if(!activeProcesses.size){process.once('SIGTERM',onTerm);process.once('SIGINT',onInt);}activeProcesses.add(abort);
|
|
62
|
+
child.stdout.on('data',data=>{stdout=append(stdout,outDecoder.write(data));options.onProgress?.();});child.stderr.on('data',data=>{stderr=append(stderr,errDecoder.write(data));options.onProgress?.();});
|
|
63
|
+
child.on('error',error=>{spawnError=error.message;done(null,null);});child.on('close',done);
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
function resolveEngine(root,id) {
|
|
67
|
+
if(!engines[id])throw Error('Choose hurl, schemathesis or k6');
|
|
68
|
+
const dir=guardedDir(home(root),path.join(home(root),id),false);
|
|
69
|
+
if(id==='hurl'){const js=path.join(dir,engines.hurl.bin);if(fs.existsSync(js))return {command:process.execPath,args:[js],managed:true};}
|
|
70
|
+
if(id==='schemathesis'){const cli=path.join(dir,process.platform==='win32'?'Scripts/schemathesis.exe':'bin/schemathesis');if(fs.existsSync(cli))return {command:cli,args:[],managed:true};}
|
|
71
|
+
if(id==='k6'){const folder='k6-v2.3.0-'+k6Assets[process.platform+'-'+process.arch]?.[0].replace(/\.(?:zip|tar\.gz)$/,'');const exe=path.join(dir,folder,process.platform==='win32'?'k6.exe':'k6');if(fs.existsSync(exe))return {command:exe,args:[],managed:true};}
|
|
72
|
+
const executable=findProgram(id);return executable?{command:executable,args:[],managed:false}:null;
|
|
73
|
+
}
|
|
74
|
+
async function doctor(root,id) {
|
|
75
|
+
const ids=id?[id]:Object.keys(engines),results=[];
|
|
76
|
+
for(const name of ids){const bin=resolveEngine(root,name);const r=bin?await execute(bin.command,[...bin.args,'--version'],{cwd:root,timeoutMs:10000}):null;results.push({engine:name,...engines[name],installed:!!r&&r.exitCode===0&&!r.error,versionOutput:r?(r.stdout+r.stderr).trim().slice(0,300):null,setup:'chode api-test setup '+name});}
|
|
77
|
+
return {ok:true,engines:results};
|
|
78
|
+
}
|
|
79
|
+
async function setup(root,id,options={}) {
|
|
80
|
+
if(!engines[id])throw Error('Choose hurl, schemathesis or k6');
|
|
81
|
+
if(resolveEngine(root,id)?.managed){const status=await doctor(root,id);if(status.engines[0].installed&&status.engines[0].versionOutput.includes(engines[id].version))return {...status,engine:id,already:true};}
|
|
82
|
+
const dir=guardedDir(home(root,true),path.join(home(root),id));let result;
|
|
83
|
+
if(id==='hurl')result=await execute(process.execPath,[npmCLI(),'install','--prefix',dir,'--no-audit','--no-fund','--no-save',engines.hurl.package],{cwd:root,timeoutMs:240000,...options});
|
|
84
|
+
else if(id==='schemathesis'){
|
|
85
|
+
const python=process.platform==='win32'?(findProgram('python')||findProgram('python3')):(findProgram('python3')||findProgram('python'));if(!python)throw Error('Python 3.10+ is required for Schemathesis setup');
|
|
86
|
+
result=await execute(python,['-m','venv',dir],{cwd:root,timeoutMs:60000});
|
|
87
|
+
if(result.exitCode===0){const py=path.join(dir,process.platform==='win32'?'Scripts/python.exe':'bin/python');result=await execute(py,['-m','pip','--disable-pip-version-check','install','schemathesis=='+engines.schemathesis.version],{cwd:root,timeoutMs:240000,...options});}
|
|
88
|
+
}else{
|
|
89
|
+
const asset=k6Assets[process.platform+'-'+process.arch];if(!asset)throw Error('Install k6 on PATH for this platform');
|
|
90
|
+
const name='k6-v2.3.0-'+asset[0],url='https://github.com/grafana/k6/releases/download/v2.3.0/'+name;
|
|
91
|
+
const response=await fetch(url,{signal:AbortSignal.timeout(120000)});if(!response.ok)throw Error('k6 download returned HTTP '+response.status);
|
|
92
|
+
const chunks=[];let bytes=0;for await(const chunk of response.body){bytes+=chunk.length;if(bytes>150*1024*1024)throw Error('k6 download exceeded limit');chunks.push(chunk);}const data=Buffer.concat(chunks);
|
|
93
|
+
if(crypto.createHash('sha256').update(data).digest('hex')!==asset[1])throw Error('k6 download checksum mismatch');
|
|
94
|
+
const archive=path.join(dir,crypto.randomUUID()+'-'+name);fs.writeFileSync(archive,data,{mode:0o600,flag:'wx'});
|
|
95
|
+
try{if(process.platform==='win32'){
|
|
96
|
+
const quote=s=>"'"+s.replace(/'/g,"''")+"'",script="$ErrorActionPreference='Stop'; Expand-Archive -LiteralPath "+quote(archive)+' -DestinationPath '+quote(dir)+' -Force';
|
|
97
|
+
result=await execute('powershell.exe',['-NoProfile','-NonInteractive','-EncodedCommand',Buffer.from(script,'utf16le').toString('base64')],{cwd:root,timeoutMs:60000});
|
|
98
|
+
}else result=await execute('tar',asset[0].endsWith('.zip')?['-xf',archive,'-C',dir]:['-xzf',archive,'-C',dir],{cwd:root,timeoutMs:60000});}finally{fs.unlinkSync(archive);}
|
|
99
|
+
}
|
|
100
|
+
if(result.exitCode!==0||result.error||result.timedOut)return {ok:false,engine:id,error:'Tool installation failed',diagnostic:require('./worker-transport').redact(result.stderr||result.stdout||result.error,root,6000)};
|
|
101
|
+
const status=await doctor(root,id);return {...status,ok:status.engines[0].installed,engine:id};
|
|
102
|
+
}
|
|
103
|
+
function integer(value,fallback,min,max,name) {if(value===undefined)return fallback;const n=Number(value);if(!Number.isInteger(n)||n<min||n>max)throw Error(name+' must be an integer from '+min+' to '+max);return n;}
|
|
104
|
+
function baseURL(value) {if(value===undefined)return undefined;const u=new URL(value);if(!['http:','https:'].includes(u.protocol)||u.username||u.password||u.search||u.hash)throw Error('Use an HTTP(S) base URL without credentials, query or fragment');return u.href.replace(/\/$/,'');}
|
|
105
|
+
async function run(root,id,file,options={}) {
|
|
106
|
+
if(!engines[id])throw Error('Choose hurl, schemathesis or k6');
|
|
107
|
+
const input=suite(root,file,id),url=baseURL(options.url),timeout=integer(options.timeout,60,1,300,'timeout'),examples=integer(options.examples,10,1,100,'examples'),iterations=integer(options.iterations,5,1,1000,'iterations'),vus=integer(options.vus,1,1,10,'vus');
|
|
108
|
+
const bin=resolveEngine(root,id);if(!bin)return {ok:false,engine:id,error:'Tool is not installed. Run chode api-test setup '+id};
|
|
109
|
+
const reportDir=guardedDir(path.resolve(root),path.join(root,'.chode','api-tests','results')),runId=crypto.randomUUID(),reportFile=path.join(reportDir,runId+'.json'),nativeReport=path.join(reportDir,runId+(id==='k6'?'.summary.json':'.junit.xml'));
|
|
110
|
+
const argv=id==='hurl'?['--test','--connect-timeout','5','--max-time','15',...(url?['--variable','base_url='+url]:[]),input.full]:id==='k6'?['run','--quiet','--no-usage-report','--vus',String(vus),'--iterations',String(iterations),'--summary-export',nativeReport,input.full]:['run',input.full,...(url?['--url',url]:[]),'--max-examples',String(examples),'--workers','1','--seed','1','--report','junit','--report-junit-path',nativeReport];
|
|
111
|
+
const version=await execute(bin.command,[...bin.args,'--version'],{cwd:root,timeoutMs:10000});if(version.exitCode!==0||version.error)throw Error('Could not verify test engine version');
|
|
112
|
+
const startedAt=new Date().toISOString(),proc=await execute(bin.command,[...bin.args,...argv],{cwd:root,timeoutMs:timeout*1000,env:url?{BASE_URL:url}:{},onProgress:options.onProgress,onAbort:()=>{try{fs.unlinkSync(nativeReport);}catch{}}});
|
|
113
|
+
function readNative(){const stat=fs.lstatSync(nativeReport);if(!stat.isFile()||stat.isSymbolicLink()||stat.nlink>1||stat.size>2*1024*1024)throw Error('Invalid native report');return fs.readFileSync(nativeReport,'utf8');}
|
|
114
|
+
let ok=proc.exitCode===0&&!proc.error&&!proc.timedOut,reason=proc.timedOut?'Test timed out':proc.error||(proc.exitCode!==0?'Test exited with code '+proc.exitCode:''),metrics=null;
|
|
115
|
+
try{if(id==='k6'){
|
|
116
|
+
const summary=JSON.parse(readNative()),metric=name=>summary.metrics?.[name]?.values||summary.metrics?.[name]||null;metrics={checks:metric('checks'),requests:metric('http_reqs'),httpFailures:metric('http_req_failed')};
|
|
117
|
+
if(metrics.checks?.fails>0||metrics.httpFailures?.passes>0){ok=false;reason='k6 recorded failed checks or HTTP requests';}
|
|
118
|
+
if(!metrics.requests?.count){ok=false;reason='k6 executed no HTTP requests';}
|
|
119
|
+
}else if(id==='schemathesis'){
|
|
120
|
+
const xml=readNative(),tests=[...xml.matchAll(/<testsuite\b[^>]*\btests="(\d+)"/g)].reduce((n,m)=>n+Number(m[1]),0);metrics={tests};if(tests===0&&ok){ok=false;reason='Schema run executed no tests';}
|
|
121
|
+
}}catch{if(ok){ok=false;reason='The test engine did not produce its required report';}}
|
|
122
|
+
try{if(crypto.createHash('sha256').update(fs.readFileSync(input.full)).digest('hex')!==input.digest){ok=false;reason='Test input changed during execution';}}catch{ok=false;reason='Test input was removed during execution';}
|
|
123
|
+
const redact=text=>require('./worker-transport').redact(text,root,16000).replace(/\x1b\[[0-?]*[ -/]*[@-~]/g,'');
|
|
124
|
+
// Native reports can echo authenticated request bodies. Retain only the redacted
|
|
125
|
+
// bounded evidence here; raw engine reports never remain on disk after the run.
|
|
126
|
+
try{fs.unlinkSync(nativeReport);}catch{}
|
|
127
|
+
const combined=redact(proc.stdout+'\n'+proc.stderr),failStart=combined.indexOf('FAILURES'),failEnd=failStart>=0?combined.indexOf('SUMMARY',failStart):0;
|
|
128
|
+
const failure=ok?'':failStart>=0?combined.slice(failStart,failEnd>failStart?failEnd:failStart+4000).slice(0,4000):combined.slice(-4000);
|
|
129
|
+
const report={ok,engine:id,toolVersion:redact(version.stdout+version.stderr).trim().slice(0,300),file:input.relative,suiteSha256:input.digest,startedAt,finishedAt:new Date().toISOString(),failure,...proc,stdout:redact(proc.stdout),stderr:redact(proc.stderr),error:redact(reason),metrics,report:path.relative(root,reportFile).replace(/\\/g,'/')};
|
|
130
|
+
fs.writeFileSync(reportFile,JSON.stringify(report,null,2),{flag:'wx',mode:0o600});return report;
|
|
131
|
+
}
|
|
132
|
+
function tokenize(text) {const tokens=[];let value='',quote=null,active=false;for(const c of String(text)){if(quote){if(c===quote)quote=null;else value+=c;}else if(c==='"'||c==="'"){quote=c;active=true;}else if(/\s/.test(c)){if(active){tokens.push(value);value='';active=false;}}else{value+=c;active=true;}}if(quote)throw Error('Close the quoted argument');if(active)tokens.push(value);return tokens;}
|
|
133
|
+
function parse(argv) {
|
|
134
|
+
const [action='list',engine,...rest]=argv;if(!['list','doctor','setup','run'].includes(action))throw Error('Use api-test list|doctor [engine]|setup engine|run engine file [--base-url URL]');
|
|
135
|
+
if(action!=='run'){if(rest.length)throw Error('Unexpected arguments');return {action,engine};}
|
|
136
|
+
const file=rest.shift(),options={};for(let i=0;i<rest.length;i+=2){const key={'--base-url':'url','--timeout':'timeout','--examples':'examples','--iterations':'iterations','--vus':'vus'}[rest[i]];if(!key||rest[i+1]===undefined)throw Error('Unknown or incomplete API-test option');options[key]=rest[i+1];}return {action,engine,file,options};
|
|
137
|
+
}
|
|
138
|
+
async function dispatch(root,argv,context={}) {const args=parse(argv);if(context.mode==='plan'&&['setup','run'].includes(args.action))throw Error('API test setup/execution is disabled in plan mode');if(context.noTools&&['setup','run'].includes(args.action))throw Error('API tests require tools');if(args.action==='list')return {ok:true,engines};if(args.action==='doctor')return doctor(root,args.engine);if(args.action==='setup')return setup(root,args.engine);return run(root,args.engine,args.file,args.options);}
|
|
139
|
+
function result(report) {return {apiTest:true,modelCalls:0,tokens:0,provider:'api-test',completionStatus:report.ok?'NOT_REQUIRED':'FAILED',result:report.engine?((report.ok?'Passed':'Failed')+' · '+report.engine+(report.file?' · '+report.file:'')+'\n'+(report.report?'Report: '+report.report+'\n':'')+(report.error||'')+'\n'+(report.failure||report.diagnostic||(report.metrics?JSON.stringify(report.metrics,null,2):JSON.stringify(report.engines||{},null,2))).slice(-6000)):JSON.stringify(report,null,2)};}
|
|
140
|
+
module.exports={engines,doctor,setup,run,dispatch,result,parse,tokenize,suite,execute,k6Assets,resolveEngine};
|
package/automatic-re.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const fs=require('fs'),path=require('path');
|
|
3
|
+
function selectTarget(prompt,root=process.cwd()){
|
|
4
|
+
const text=String(prompt||'');if(!/reverse[ -]?engineer|\b(?:analy[sz]e|inspect|decompile|disassemble|binary|executable)\b|what.{0,30}\bdoes\b|how.{0,30}\bworks?\b/i.test(text))return null;
|
|
5
|
+
if(/\b(?:do not|don't|skip|avoid)\s+(?:\w+\s+){0,3}(?:inspect|analy[sz]e|decompile|disassemble|reverse)/i.test(text))return null;
|
|
6
|
+
const candidates=[];const rx=/["`]([^"`\r\n]+\.(?:exe|dll|elf|so|dylib))["`]|(?:^|\s)@?([^\s"`]+\.(?:exe|dll|elf|so|dylib))(?=$|[\s,;?!])/gi;let m;
|
|
7
|
+
while((m=rx.exec(text))){const candidate=m[1]||m[2];if(/^[a-z]+:\/\//i.test(candidate)||candidate.startsWith('\\\\'))continue;const full=path.resolve(root,candidate);try{if(fs.statSync(full).isFile()&&!candidates.includes(full))candidates.push(full);}catch(_){}}
|
|
8
|
+
return candidates.length===1?candidates[0]:null;
|
|
9
|
+
}
|
|
10
|
+
async function prepare(prompt,options={}){
|
|
11
|
+
if(options.disabled)return null;const target=selectTarget(prompt,options.root);if(!target)return null;
|
|
12
|
+
const call={tool:'rea',args:{action:'analyze',path:target,timeoutMs:60000}};
|
|
13
|
+
const emit=event=>{try{if(options.onTool)options.onTool(event);}catch(_){}};
|
|
14
|
+
emit({phase:'start',call});let result;try{result=await options.analyze(call.args);}catch(_){result={ok:false};}emit({phase:'done',call,result});
|
|
15
|
+
if(!result||!result.ok)return {target,ok:false,context:'[LOCAL BINARY INSPECTION]\nAutomatic local analysis was unavailable. Do not claim this executable was inspected or its behavior recovered. Suggest chode tool rea doctor for setup diagnostics.\n[END LOCAL BINARY INSPECTION]'};
|
|
16
|
+
const ev=result.evidence||{};const facts={};for(const k of ['digest','format','arch','segments','procedures','strings'])if(ev[k]!==undefined)facts[k]=ev[k];
|
|
17
|
+
return {target,ok:true,evidence:facts,context:'[LOCAL BINARY INSPECTION — tool-observed metadata, not executable behavior]\n'+JSON.stringify(facts)+'\nStatic inventory does not recover business logic. Do not invent functions, behavior or decompilation. Use the rea tool for further supported inspection; original executable was not run.\n[END LOCAL BINARY INSPECTION]'};
|
|
18
|
+
}
|
|
19
|
+
module.exports={selectTarget,prepare};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": 1,
|
|
3
|
+
"retrievedOn": "2026-10-17",
|
|
4
|
+
"chode": { "publishedScore": null, "status": "unmeasured", "supremacyEstablished": false },
|
|
5
|
+
"baselines": [
|
|
6
|
+
{ "benchmark": "Terminal-Bench", "version": "4.0", "agent": "Claude Code", "model": "Opus 5.5", "effort": "max", "score": 64.8, "unit": "resolution percent", "confidence95PlusMinus": 3.1, "source": "https://www.tbench.ai/", "sourceType": "official leaderboard", "observation": "Hydrated table selected 4.0; leaderboard display names preserved." },
|
|
7
|
+
{ "benchmark": "Terminal-Bench", "version": "2.0", "agent": "Claude Code", "model": "Claude Opus 4.5", "score": 52.1, "unit": "resolution percent", "confidence95PlusMinus": 2.5, "source": "https://www.tbench.ai/?version=2.0", "sourceType": "official leaderboard", "observation": "Older matched-model comparator; not comparable to 4.0." },
|
|
8
|
+
{ "benchmark": "Terminal-Bench", "version": "2.0", "agent": "OpenCode", "model": "Claude Opus 4.5", "score": 51.7, "unit": "resolution percent", "confidence95PlusMinus": null, "source": "https://www.tbench.ai/?version=2.0", "sourceType": "official leaderboard", "observation": "No uncertainty interval supplied; do not infer a statistically significant gap." },
|
|
9
|
+
{ "benchmark": "WebDev Arena", "version": "2026-10-16 leaderboard", "agent": null, "model": "Claude Opus 5.5 (Max)", "score": 1807, "unit": "Arena score", "confidence95PlusMinus": 14, "source": "https://arena.ai/leaderboard/code/webdev", "sourceType": "official leaderboard", "observation": "Model/build preference score. This is not a Claude Code or OpenCode harness score." }
|
|
10
|
+
],
|
|
11
|
+
"researchReference": {
|
|
12
|
+
"benchmark": "DeepResearch Bench II",
|
|
13
|
+
"source": "https://agentresearchlab.com/benchmarks/deepresearch-bench-ii/",
|
|
14
|
+
"currentLeaderboard": "https://huggingface.co/spaces/muset-ai/DeepResearch-Bench-Leaderboard",
|
|
15
|
+
"tasks": 132,
|
|
16
|
+
"rubrics": 9430,
|
|
17
|
+
"dimensions": ["Information Recall", "Analysis", "Presentation"],
|
|
18
|
+
"score": null,
|
|
19
|
+
"note": "The project page marks its numeric table as superseded and announces a changed judge. Do not use that older table as a current numeric target."
|
|
20
|
+
},
|
|
21
|
+
"comparisonPolicy": [
|
|
22
|
+
"A public competitor score is a target, never a score assigned to Chode.",
|
|
23
|
+
"Compare only matching benchmark versions, task sets and evaluation protocols.",
|
|
24
|
+
"Separate model performance from harness performance; control model and budget for a harness claim.",
|
|
25
|
+
"Preserve uncertainty, missing intervals, dated snapshots, and failed or unavailable runs.",
|
|
26
|
+
"No output-clarity or TUI-superiority claim follows from these benchmarks.",
|
|
27
|
+
"Do not make a broad superiority claim from a small synthetic smoke test."
|
|
28
|
+
]
|
|
29
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const snapshot=require('./public-baselines.json');
|
|
3
|
+
function compare(result,baseline){
|
|
4
|
+
const required=['benchmark','version','unit','protocol'];
|
|
5
|
+
if(required.some(k=>!result[k])||!Number.isFinite(result.score))return {comparable:false,reason:'A measured Chode result and evaluation protocol are required.'};
|
|
6
|
+
if(['benchmark','version','unit'].some(k=>result[k]!==baseline[k]))return {comparable:false,reason:'Benchmark version or metric differs.'};
|
|
7
|
+
if(result.protocol!==baseline.protocol||!baseline.protocol)return {comparable:false,reason:'Matching evaluation protocol has not been recorded.'};
|
|
8
|
+
return {comparable:true,pointEstimateHigher:result.score>baseline.score,reason:'Point estimates alone do not establish statistical superiority.'};
|
|
9
|
+
}
|
|
10
|
+
if(require.main===module){for(const b of snapshot.baselines)console.log(`${b.benchmark} ${b.version}: ${b.agent||'model benchmark'} / ${b.model}: ${b.score} ${b.unit}\n ${b.source}`);console.log('\nChode: unmeasured on these public benchmarks.');}
|
|
11
|
+
module.exports={snapshot,compare};
|