@riddledc/riddle-proof 0.8.66 → 0.8.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -261,6 +261,17 @@ is generated only as wrapper/runtime validation guidance; the browser evidence,
261
261
  required cases, forbidden lifecycle markers, and version gate remain owned by
262
262
  the generic pack manifest.
263
263
 
264
+ For broader dogfood loops, keep a bounded story matrix instead of asking an
265
+ agent to sweep every feature at once. The starter matrix lives at
266
+ `examples/story-matrices/riddle-proof-bounded-loop.json`; see
267
+ `examples/story-matrices/README.md` for the package-local loop notes and
268
+ `docs/riddle-proof-story-matrix.md` in the repository for the longer workflow.
269
+ Each story names its surface, expected behavior, runner, evidence receipts,
270
+ likely failure classes, and whether it is candidate, ready, covered by existing
271
+ tests, blocked by external infrastructure, or recurring. Treat negative
272
+ controls as successful framework stories when they return the expected non-pass
273
+ verdict with evidence.
274
+
264
275
  Before counting live wrapper runs, use the pack's runtime gate: verify
265
276
  `riddle_proof_status` reports the loaded `@riddledc/openclaw-riddle-proof` and
266
277
  `@riddledc/riddle-proof` versions. Disk package versions alone are not enough.
@@ -0,0 +1,17 @@
1
+ # Riddle Proof Story Matrices
2
+
3
+ `riddle-proof-bounded-loop.json` is a starter matrix for bounded dogfood loops.
4
+ It is meant to keep broad user-story sweeps evidence-backed and reviewable
5
+ without turning them into unbounded scaffolding.
6
+
7
+ Each story names:
8
+
9
+ - the Riddle Proof surface under test
10
+ - the expected behavior and expected verdict
11
+ - the runner to use
12
+ - the concrete receipts that must be saved
13
+ - likely failure classes
14
+ - the story status, such as candidate, ready, covered, blocked, or recurring
15
+
16
+ Use one batch at a time, save outputs under `artifacts/riddle-proof/<story-id>/`,
17
+ fix only high-signal failures, and rerun the same batch before expanding scope.
@@ -0,0 +1,299 @@
1
+ {
2
+ "version": "riddle-proof.story-matrix.v1",
3
+ "matrix_id": "riddle-proof-bounded-loop-2026-06",
4
+ "public_name": "Riddle Proof Bounded Dogfood Loop",
5
+ "description": "A small canonical matrix for exercising Riddle Proof as its own proving-ground product before scaling to broad generated user-story sweeps.",
6
+ "operating_rules": [
7
+ "Keep each pass bounded: choose one batch, run it, classify findings, fix only high-signal issues, then rerun the same batch.",
8
+ "A story is not passed until it records concrete evidence receipts such as command output, profile-result.json, proof.json, screenshots, PR comment body, or a formal/test artifact.",
9
+ "Do not count GitHub Actions failures caused by billing or unavailable runners as product regressions. Classify those as environment_blocked.",
10
+ "Prefer generic Riddle Proof surfaces over adapter-specific wrapper behavior unless the story explicitly names an adapter.",
11
+ "Negative-control stories are first-class: a correct product_regression or proof_insufficient result can be a pass for the framework."
12
+ ],
13
+ "batch_policy": {
14
+ "default_batch_size": 8,
15
+ "max_batch_size": 20,
16
+ "rerun_rule": "After fixing a product or contract issue found by the batch, rerun every story in the same batch before advancing.",
17
+ "promotion_rule": "Promote a story from candidate to recurring only after it has at least one saved evidence packet and a stable expected verdict."
18
+ },
19
+ "status_values": [
20
+ "candidate",
21
+ "ready_to_run",
22
+ "covered_existing",
23
+ "blocked_external",
24
+ "needs_product_hook",
25
+ "recurring"
26
+ ],
27
+ "failure_classes": [
28
+ "product_regression",
29
+ "proof_insufficient",
30
+ "environment_blocked",
31
+ "configuration_error",
32
+ "needs_human_review",
33
+ "contract_gap",
34
+ "doc_or_ux_gap"
35
+ ],
36
+ "evidence_receipt_types": [
37
+ "profile-result.json",
38
+ "proof.json",
39
+ "summary.md",
40
+ "screenshot",
41
+ "console.json",
42
+ "dom-summary.json",
43
+ "regression-pack-result.json",
44
+ "pr-comment-body.md",
45
+ "preflight-json",
46
+ "formal-kernel-build",
47
+ "runtime-test-log",
48
+ "manual-review-note"
49
+ ],
50
+ "stories": [
51
+ {
52
+ "id": "rp-story-profile-suggest-drafts-not-verdicts",
53
+ "batch": "authoring",
54
+ "surface": "profile-suggest",
55
+ "user_story": "As a proof author, I can ask Riddle Proof to suggest checks for a changed route without accidentally treating suggestions as proof.",
56
+ "expected_behavior": "The command emits a draft profile or JSON suggestion and clearly does not produce a proof verdict.",
57
+ "primary_runner": "local-cli",
58
+ "command": "riddle-proof-loop profile-suggest --route /proof --changed-files docs/proof.md --selectors main#main-content --format profile",
59
+ "expected_verdict": "draft_only",
60
+ "evidence_required": [
61
+ "generated profile includes route_loaded or selector checks",
62
+ "output contains no passed/product_regression/proof_insufficient verdict",
63
+ "operator records the saved draft path or stdout capture"
64
+ ],
65
+ "receipt_types": ["manual-review-note"],
66
+ "failure_classes": ["doc_or_ux_gap", "contract_gap"],
67
+ "status": "candidate"
68
+ },
69
+ {
70
+ "id": "rp-story-hosted-happy-path-produces-artifacts",
71
+ "batch": "hosted-profile",
72
+ "surface": "run-profile hosted runner",
73
+ "user_story": "As a product maintainer, I can run a hosted profile against a live route and receive a passed verdict tied to screenshots and structured artifacts.",
74
+ "expected_behavior": "Hosted run-profile returns passed, writes profile-result.json, summary.md, proof.json, console.json, dom-summary.json, and records screenshot artifact names.",
75
+ "primary_runner": "hosted-riddle",
76
+ "command": "riddle-proof-loop run-profile --profile <route-smoke-profile> --url https://riddledc.com --runner riddle --output artifacts/riddle-proof/<story-id> --result-format compact-json",
77
+ "expected_verdict": "passed",
78
+ "evidence_required": [
79
+ "profile-result.json status is passed",
80
+ "riddle.job_id is present",
81
+ "proof_json and screenshot artifacts are listed"
82
+ ],
83
+ "receipt_types": ["profile-result.json", "proof.json", "summary.md", "screenshot"],
84
+ "failure_classes": ["product_regression", "proof_insufficient", "environment_blocked"],
85
+ "existing_coverage": ["examples/regression-packs/oc-flow-regression.json: hosted-home-to-proof-route-change-pass"],
86
+ "status": "covered_existing"
87
+ },
88
+ {
89
+ "id": "rp-story-hosted-negative-control-stays-regression",
90
+ "batch": "hosted-profile",
91
+ "surface": "run-profile hosted runner",
92
+ "user_story": "As a maintainer, I can run a negative-control profile and see a product_regression instead of a false pass.",
93
+ "expected_behavior": "A missing selector or missing query parameter produces product_regression with evidence, not passed and not a lifecycle crash.",
94
+ "primary_runner": "hosted-riddle",
95
+ "command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core false --hosted-riddle true --format markdown --output artifacts/riddle-proof/hosted-regression",
96
+ "expected_verdict": "product_regression",
97
+ "evidence_required": [
98
+ "negative case expected status is product_regression",
99
+ "profile-result.json includes the failed check",
100
+ "summary names the failed invariant without hiding artifacts"
101
+ ],
102
+ "receipt_types": ["profile-result.json", "summary.md", "console.json"],
103
+ "failure_classes": ["contract_gap", "proof_insufficient", "environment_blocked"],
104
+ "existing_coverage": [
105
+ "examples/regression-packs/oc-flow-regression.json: hosted-pricing-query-hash-dropped-blocker",
106
+ "examples/regression-packs/oc-flow-regression.json: hosted-missing-selector-timeout-blocker"
107
+ ],
108
+ "status": "covered_existing"
109
+ },
110
+ {
111
+ "id": "rp-story-cold-start-does-not-duplicate-too-soon",
112
+ "batch": "hosted-profile",
113
+ "surface": "hosted worker cold start",
114
+ "user_story": "As a hosted runner user, I can submit work while workers are scaled to zero without the client declaring the job stale before normal cold-start readiness.",
115
+ "expected_behavior": "The unsubmitted retry window is long enough for accepted scale-to-zero cold starts, and progress output explains worker-wake waiting.",
116
+ "primary_runner": "hosted-riddle",
117
+ "command": "riddle-proof-loop run-profile --profile <small-profile> --url <site> --runner riddle --output artifacts/riddle-proof/cold-start-smoke --result-format compact-json --progress-every-ms 5000",
118
+ "expected_verdict": "passed",
119
+ "evidence_required": [
120
+ "attempts budget corresponds to a five-minute unsubmitted timeout",
121
+ "progress output reports waiting_for_worker_submit before submission",
122
+ "profile-result.json passes or returns environment_blocked with the specific infra blocker"
123
+ ],
124
+ "receipt_types": ["profile-result.json", "summary.md", "manual-review-note"],
125
+ "failure_classes": ["environment_blocked", "contract_gap"],
126
+ "existing_coverage": ["README cold-start retry documentation", "cli default timeout test"],
127
+ "status": "recurring"
128
+ },
129
+ {
130
+ "id": "rp-story-local-core-trust-boundary",
131
+ "batch": "local-core",
132
+ "surface": "runtime trust boundary",
133
+ "user_story": "As a framework maintainer, I can run the generic local core suite and prove route, query/hash, thrown-error, missing-selector, and no-diff cases without hosted infrastructure.",
134
+ "expected_behavior": "The local core regression suite passes all required cases and reports no forbidden terminal markers.",
135
+ "primary_runner": "local-core",
136
+ "command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle false --format compact-json --output artifacts/riddle-proof/local-core",
137
+ "expected_verdict": "passed",
138
+ "evidence_required": [
139
+ "regression-pack-result.json local_core.ok is true",
140
+ "missing_required_cases is empty",
141
+ "forbidden_terminal_markers_seen is empty"
142
+ ],
143
+ "receipt_types": ["regression-pack-result.json", "runtime-test-log"],
144
+ "failure_classes": ["contract_gap", "configuration_error"],
145
+ "existing_coverage": ["runtime/tests/trust_boundary_regression.py"],
146
+ "status": "recurring"
147
+ },
148
+ {
149
+ "id": "rp-story-pr-comment-renders-only-evidence-backed-status",
150
+ "batch": "reporting",
151
+ "surface": "pr-comment",
152
+ "user_story": "As a reviewer, I can get a PR comment whose badge, artifact links, and failed checks reflect the saved proof packet rather than raw guessed fields.",
153
+ "expected_behavior": "Dry-run PR comment generation renders status and artifacts from proof outputs and refuses missing required inputs.",
154
+ "primary_runner": "local-cli",
155
+ "command": "riddle-proof-loop pr-comment --proof-dir artifacts/riddle-proof/<story-id> --pr <number> --repo <owner/repo> --dry-run --body-file artifacts/riddle-proof/<story-id>/pr-comment-body.md",
156
+ "expected_verdict": "passed",
157
+ "evidence_required": [
158
+ "body file includes the same status as profile-result.json",
159
+ "required artifact links or local names are present",
160
+ "missing proof-dir inputs fail rather than rendering passed"
161
+ ],
162
+ "receipt_types": ["pr-comment-body.md", "profile-result.json"],
163
+ "failure_classes": ["contract_gap", "doc_or_ux_gap"],
164
+ "existing_coverage": ["test.js pr-comment cases"],
165
+ "status": "ready_to_run"
166
+ },
167
+ {
168
+ "id": "rp-story-profile-body-assertions-derive-from-real-artifact",
169
+ "batch": "artifact-publication",
170
+ "surface": "profile-body-assertions",
171
+ "user_story": "As a profile promoter, I can derive body assertions from a real artifact before making a public proof/profile claim.",
172
+ "expected_behavior": "The command emits snippets only when they are present in the artifact and fails when required fragments are missing.",
173
+ "primary_runner": "local-cli",
174
+ "command": "riddle-proof-loop profile-body-assertions --artifact <proof-json-or-url> --candidates-json '[\"product_regression\",\"passed\"]' --required-json '[\"product_regression\"]' --format body-contains",
175
+ "expected_verdict": "passed_or_failed_by_artifact",
176
+ "evidence_required": [
177
+ "required present fragment is emitted",
178
+ "required missing fragment exits nonzero",
179
+ "promoted profile records derived snippets instead of hand-authored guesses"
180
+ ],
181
+ "receipt_types": ["preflight-json", "manual-review-note"],
182
+ "failure_classes": ["contract_gap", "proof_insufficient"],
183
+ "existing_coverage": ["README profile-body-assertions guidance", "test.js body assertion cases"],
184
+ "status": "ready_to_run"
185
+ },
186
+ {
187
+ "id": "rp-story-http-preflight-catches-publication-mismatch",
188
+ "batch": "artifact-publication",
189
+ "surface": "profile-http-status-preflight",
190
+ "user_story": "As a public proof publisher, I can check artifact URLs and body fragments before spending a hosted browser run.",
191
+ "expected_behavior": "The preflight returns success only when status, content type, byte size, required body text, and forbidden body text all satisfy the profile.",
192
+ "primary_runner": "local-cli",
193
+ "command": "riddle-proof-loop profile-http-status-preflight --profile <public-promotion-profile> --url <preview-url> --format json",
194
+ "expected_verdict": "passed_or_product_regression",
195
+ "evidence_required": [
196
+ "preflight output lists every checked URL",
197
+ "missing required body text fails",
198
+ "forbidden escaped/raw artifact text fails"
199
+ ],
200
+ "receipt_types": ["preflight-json"],
201
+ "failure_classes": ["product_regression", "proof_insufficient", "configuration_error"],
202
+ "existing_coverage": ["test.js http preflight cases"],
203
+ "status": "ready_to_run"
204
+ },
205
+ {
206
+ "id": "rp-story-run-profile-recover-preserves-artifacts",
207
+ "batch": "hosted-profile",
208
+ "surface": "run-profile recover",
209
+ "user_story": "As a user whose local process was interrupted, I can recover a hosted job and still get the proof artifacts and summary.",
210
+ "expected_behavior": "Recover fetches the hosted job artifacts, writes the same profile-result/summary/proof files, and preserves the original job id.",
211
+ "primary_runner": "hosted-riddle",
212
+ "command": "riddle-proof-loop run-profile recover --profile <profile> --url <base-url> --job <job-id> --output artifacts/riddle-proof/<story-id>-recovered --result-format compact-json",
213
+ "expected_verdict": "passed_or_original_verdict",
214
+ "evidence_required": [
215
+ "recovered profile-result.json includes the requested job id",
216
+ "summary.md is written",
217
+ "proof.json and screenshots are recovered or explicitly reported missing"
218
+ ],
219
+ "receipt_types": ["profile-result.json", "summary.md", "proof.json", "screenshot"],
220
+ "failure_classes": ["environment_blocked", "proof_insufficient", "contract_gap"],
221
+ "status": "candidate"
222
+ },
223
+ {
224
+ "id": "rp-story-viewport-aggregate-does-not-hide-child-failure",
225
+ "batch": "hosted-profile",
226
+ "surface": "run-profile aggregate",
227
+ "user_story": "As a maintainer running viewport matrices, I can aggregate child runs without losing the failing viewport or overstating total latency.",
228
+ "expected_behavior": "Aggregate returns the worst required child verdict, lists child outputs, and separates summed poll time from max child latency.",
229
+ "primary_runner": "local-cli",
230
+ "command": "riddle-proof-loop run-profile aggregate --profile <matrix-profile> --url <base-url> --input-dir artifacts/riddle-proof/<matrix-dir> --output artifacts/riddle-proof/<matrix-dir>",
231
+ "expected_verdict": "worst_child_verdict",
232
+ "evidence_required": [
233
+ "aggregate profile-result.json names all child viewports",
234
+ "any failed child remains visible in checks",
235
+ "summary reports child timing without implying a single hosted job"
236
+ ],
237
+ "receipt_types": ["profile-result.json", "summary.md"],
238
+ "failure_classes": ["contract_gap", "proof_insufficient"],
239
+ "existing_coverage": ["test.js aggregate cases"],
240
+ "status": "ready_to_run"
241
+ },
242
+ {
243
+ "id": "rp-story-formal-kernel-matches-runtime-contract",
244
+ "batch": "formal-contract",
245
+ "surface": "formal/riddle-proof-kernel",
246
+ "user_story": "As a framework maintainer, I can prove the result/status contract that runtime tests depend on, without putting Lean in the evidence-collection hot path.",
247
+ "expected_behavior": "The formal kernel builds, and conformance tests map the same runtime cases into executable checks.",
248
+ "primary_runner": "lean-kernel",
249
+ "command": "cd formal/riddle-proof-kernel && lake build && cd ../.. && node packages/riddle-proof/formal-conformance.test.js",
250
+ "expected_verdict": "passed",
251
+ "evidence_required": [
252
+ "lake build exits zero",
253
+ "formal-conformance.test.js exits zero",
254
+ "new runtime status cases are either mirrored in Lean or explicitly classified as out of formal scope"
255
+ ],
256
+ "receipt_types": ["formal-kernel-build", "runtime-test-log"],
257
+ "failure_classes": ["contract_gap", "configuration_error"],
258
+ "existing_coverage": ["formal/riddle-proof-kernel", "formal-conformance.test.js"],
259
+ "status": "recurring"
260
+ },
261
+ {
262
+ "id": "rp-story-agent-loop-checkpoint-stale-response",
263
+ "batch": "agent-loop",
264
+ "surface": "checkpoint/respond/status",
265
+ "user_story": "As a host integration, I can ignore a stale checkpoint response after the run has already reached a terminal status.",
266
+ "expected_behavior": "A late response is recorded as ignored and does not reopen the run or start background resume.",
267
+ "primary_runner": "local-core",
268
+ "command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle false --format json --output artifacts/riddle-proof/checkpoint-stale",
269
+ "expected_verdict": "passed",
270
+ "evidence_required": [
271
+ "late-stale-checkpoint-ignored case is present",
272
+ "ignored_checkpoint_response is true",
273
+ "background_resume_started is false"
274
+ ],
275
+ "receipt_types": ["regression-pack-result.json", "runtime-test-log"],
276
+ "failure_classes": ["contract_gap"],
277
+ "existing_coverage": ["examples/regression-packs/oc-flow-regression.json: late-stale-checkpoint-ignored"],
278
+ "status": "covered_existing"
279
+ },
280
+ {
281
+ "id": "rp-story-hosted-proof-view-uses-contract",
282
+ "batch": "hosted-proof-view",
283
+ "surface": "hosted proof renderer",
284
+ "user_story": "As a viewer of a hosted proof page, I see the contract-derived status, artifacts, and missing-evidence explanation rather than raw backend fields.",
285
+ "expected_behavior": "Hosted proof views and any agent-facing summaries import/use the same status/artifact contract as PR comments.",
286
+ "primary_runner": "manual-review",
287
+ "command": "Inspect hosted proof renderer imports and run a hosted proof view smoke with a passed and a negative-control packet.",
288
+ "expected_verdict": "needs_human_review",
289
+ "evidence_required": [
290
+ "renderer uses shared contract code or has a documented gap",
291
+ "passed packet renders passed with artifacts",
292
+ "negative-control packet renders product_regression or proof_insufficient, never passed"
293
+ ],
294
+ "receipt_types": ["manual-review-note", "screenshot", "profile-result.json"],
295
+ "failure_classes": ["contract_gap", "doc_or_ux_gap"],
296
+ "status": "needs_product_hook"
297
+ }
298
+ ]
299
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@riddledc/riddle-proof",
3
- "version": "0.8.66",
3
+ "version": "0.8.67",
4
4
  "description": "Reusable Riddle Proof contracts and helpers for evidence-backed agent changes.",
5
5
  "license": "MIT",
6
6
  "author": "RiddleDC",
@@ -247,6 +247,6 @@
247
247
  "build": "npm --workspace @riddledc/riddle-proof-app-contract run build --if-present && tsup src/index.ts src/types.ts src/result.ts src/state.ts src/checkpoint.ts src/run-card.ts src/public-state.ts src/runner.ts src/engine-harness.ts src/codex-exec-agent.ts src/local-agent.ts src/cli.ts src/cli/index.ts src/diagnostics.ts src/proof-session.ts src/playability.ts src/basic-gameplay.ts src/profile.ts src/profile/index.ts src/profile-suggestions.ts src/pr-comment.ts src/openclaw.ts src/proof-run-core.ts src/proof-run-engine.ts src/riddle-client.ts src/runtime/riddle-client.ts src/spec/index.ts src/spec/types.ts src/spec/result.ts src/spec/state.ts src/spec/checkpoint.ts src/spec/run-card.ts src/spec/public-state.ts src/runtime/index.ts src/app-contract/index.ts src/advanced/index.ts src/advanced/runner.ts src/advanced/engine-harness.ts src/advanced/proof-run-core.ts src/advanced/proof-run-engine.ts src/adapters/openclaw.ts src/adapters/local-agent.ts src/adapters/codex-exec-agent.ts src/adapters/codex.ts --format cjs,esm --dts --out-dir dist --clean",
248
248
  "clean": "rm -rf dist",
249
249
  "lint": "echo 'lint: (not configured)'",
250
- "test": "npm run build && node test.js && node proof-run.test.js && node formal-conformance.test.js && node trust-boundary.test.js && node regression-packs.test.js && python3 runtime/tests/trust_boundary_regression.py && python3 runtime/tests/ship_artifact_publication.py && (python3 runtime/tests/recon_verify_smoke.py >/tmp/riddle-proof-recon-verify-smoke.json || (tail -120 /tmp/riddle-proof-recon-verify-smoke.json; exit 1))"
250
+ "test": "npm run build && node test.js && node proof-run.test.js && node formal-conformance.test.js && node trust-boundary.test.js && node regression-packs.test.js && node story-matrix.test.js && python3 runtime/tests/trust_boundary_regression.py && python3 runtime/tests/ship_artifact_publication.py && (python3 runtime/tests/recon_verify_smoke.py >/tmp/riddle-proof-recon-verify-smoke.json || (tail -120 /tmp/riddle-proof-recon-verify-smoke.json; exit 1))"
251
251
  }
252
252
  }