@riddledc/riddle-proof 0.8.66 → 0.8.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/{chunk-VSP4K5LR.js → chunk-AF3KFD4A.js} +10 -5
- package/dist/{chunk-RZ3GXSXQ.js → chunk-DTFSHRV5.js} +76 -7
- package/dist/cli/index.js +2 -2
- package/dist/cli.cjs +85 -11
- package/dist/cli.js +2 -2
- package/dist/index.cjs +76 -7
- package/dist/index.js +1 -1
- package/dist/pr-comment.cjs +76 -7
- package/dist/pr-comment.js +1 -1
- package/examples/story-matrices/README.md +17 -0
- package/examples/story-matrices/riddle-proof-bounded-loop.json +299 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -261,6 +261,17 @@ is generated only as wrapper/runtime validation guidance; the browser evidence,
|
|
|
261
261
|
required cases, forbidden lifecycle markers, and version gate remain owned by
|
|
262
262
|
the generic pack manifest.
|
|
263
263
|
|
|
264
|
+
For broader dogfood loops, keep a bounded story matrix instead of asking an
|
|
265
|
+
agent to sweep every feature at once. The starter matrix lives at
|
|
266
|
+
`examples/story-matrices/riddle-proof-bounded-loop.json`; see
|
|
267
|
+
`examples/story-matrices/README.md` for the package-local loop notes and
|
|
268
|
+
`docs/riddle-proof-story-matrix.md` in the repository for the longer workflow.
|
|
269
|
+
Each story names its surface, expected behavior, runner, evidence receipts,
|
|
270
|
+
likely failure classes, and whether it is candidate, ready, covered by existing
|
|
271
|
+
tests, blocked by external infrastructure, or recurring. Treat negative
|
|
272
|
+
controls as successful framework stories when they return the expected non-pass
|
|
273
|
+
verdict with evidence.
|
|
274
|
+
|
|
264
275
|
Before counting live wrapper runs, use the pack's runtime gate: verify
|
|
265
276
|
`riddle_proof_status` reports the loaded `@riddledc/openclaw-riddle-proof` and
|
|
266
277
|
`@riddledc/riddle-proof` versions. Disk package versions alone are not enough.
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
RIDDLE_PROOF_PR_COMMENT_MARKER,
|
|
9
9
|
buildRiddleProofPrCommentMarkdown,
|
|
10
10
|
summarizeRiddleProofPrComment
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-DTFSHRV5.js";
|
|
12
12
|
import {
|
|
13
13
|
suggestRiddleProofProfileChecks
|
|
14
14
|
} from "./chunk-5Y4V2IXI.js";
|
|
@@ -4713,12 +4713,17 @@ async function main() {
|
|
|
4713
4713
|
}
|
|
4714
4714
|
if (command === "pr-comment") {
|
|
4715
4715
|
const proofDir = optionString(options, "proofDir") || optionString(options, "outputDir") || positional[1];
|
|
4716
|
-
const
|
|
4717
|
-
const
|
|
4716
|
+
const explicitRunResponsePath = optionString(options, "runResponse");
|
|
4717
|
+
const explicitResultJsonPath = optionString(options, "resultJson");
|
|
4718
|
+
const runResponsePath = explicitRunResponsePath || defaultProofDirJsonPath(proofDir, "riddle-run-response.json");
|
|
4719
|
+
const resultJsonPaths = explicitResultJsonPath ? [explicitResultJsonPath] : [
|
|
4720
|
+
defaultProofDirJsonPath(proofDir, "result.json"),
|
|
4721
|
+
defaultProofDirJsonPath(proofDir, "profile-result.json")
|
|
4722
|
+
];
|
|
4718
4723
|
const runResponse = readJsonFileIfExists(runResponsePath);
|
|
4719
|
-
const result = readJsonFileIfExists(
|
|
4724
|
+
const result = resultJsonPaths.map((candidate) => readJsonFileIfExists(candidate)).find(Boolean);
|
|
4720
4725
|
if (!runResponse && !result) {
|
|
4721
|
-
throw new Error("pr-comment requires --proof-dir with riddle-run-response.json
|
|
4726
|
+
throw new Error("pr-comment requires --proof-dir with riddle-run-response.json, result.json, or profile-result.json, or explicit --run-response/--result-json.");
|
|
4722
4727
|
}
|
|
4723
4728
|
const body = buildRiddleProofPrCommentMarkdown({
|
|
4724
4729
|
title: optionString(options, "title"),
|
|
@@ -59,6 +59,36 @@ function collectArtifacts(runResponse) {
|
|
|
59
59
|
}
|
|
60
60
|
return artifacts;
|
|
61
61
|
}
|
|
62
|
+
function collectProfileArtifacts(result) {
|
|
63
|
+
const resultArtifacts = asRecord(result.artifacts);
|
|
64
|
+
const artifacts = [];
|
|
65
|
+
const seen = /* @__PURE__ */ new Set();
|
|
66
|
+
for (const [index, item] of asArray(resultArtifacts.riddle_artifacts).entries()) {
|
|
67
|
+
const artifact = asRecord(item);
|
|
68
|
+
const url = stringValue(artifact.url);
|
|
69
|
+
if (!url || seen.has(url)) continue;
|
|
70
|
+
seen.add(url);
|
|
71
|
+
const fallbackName = stringValue(artifact.kind) || `artifact-${index + 1}`;
|
|
72
|
+
const name = artifactDisplayName(artifact.name, fallbackName);
|
|
73
|
+
artifacts.push({
|
|
74
|
+
name,
|
|
75
|
+
url,
|
|
76
|
+
kind: artifactKind(name, url),
|
|
77
|
+
size_bytes: numberValue(artifact.size_bytes) ?? numberValue(artifact.size)
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
return artifacts;
|
|
81
|
+
}
|
|
82
|
+
function mergeArtifacts(...artifactLists) {
|
|
83
|
+
const artifacts = [];
|
|
84
|
+
const seen = /* @__PURE__ */ new Set();
|
|
85
|
+
for (const artifact of artifactLists.flat()) {
|
|
86
|
+
if (seen.has(artifact.url)) continue;
|
|
87
|
+
seen.add(artifact.url);
|
|
88
|
+
artifacts.push(artifact);
|
|
89
|
+
}
|
|
90
|
+
return artifacts;
|
|
91
|
+
}
|
|
62
92
|
function pageSummaries(result) {
|
|
63
93
|
const pages = [];
|
|
64
94
|
for (const page of asArray(result.pages)) {
|
|
@@ -75,6 +105,35 @@ function pageSummaries(result) {
|
|
|
75
105
|
}
|
|
76
106
|
return pages;
|
|
77
107
|
}
|
|
108
|
+
function profileCheckCounts(result) {
|
|
109
|
+
const checks = asArray(result.checks);
|
|
110
|
+
if (checks.length) {
|
|
111
|
+
let passed2 = 0;
|
|
112
|
+
let failed2 = 0;
|
|
113
|
+
for (const item of checks) {
|
|
114
|
+
const status = stringValue(asRecord(item).status);
|
|
115
|
+
if (status === "passed") passed2 += 1;
|
|
116
|
+
if (status === "failed") failed2 += 1;
|
|
117
|
+
}
|
|
118
|
+
return { passed: passed2, failed: failed2 };
|
|
119
|
+
}
|
|
120
|
+
const checkCounts = asRecord(result.check_counts);
|
|
121
|
+
const passed = numberValue(checkCounts.passed);
|
|
122
|
+
const failed = numberValue(checkCounts.failed);
|
|
123
|
+
if (typeof passed === "number" || typeof failed === "number") {
|
|
124
|
+
return { passed: passed ?? 0, failed: failed ?? 0 };
|
|
125
|
+
}
|
|
126
|
+
return void 0;
|
|
127
|
+
}
|
|
128
|
+
function profilePageSummaries(result, counts) {
|
|
129
|
+
if (!counts) return [];
|
|
130
|
+
const route = asRecord(result.route);
|
|
131
|
+
return [{
|
|
132
|
+
route: firstStringValue(route.observed, route.expected_path, route.requested, result.profile_name) || "profile",
|
|
133
|
+
passed: counts.passed,
|
|
134
|
+
failed: counts.failed
|
|
135
|
+
}];
|
|
136
|
+
}
|
|
78
137
|
function summarizeExplicitChecks(value) {
|
|
79
138
|
let passed = 0;
|
|
80
139
|
let failed = 0;
|
|
@@ -126,18 +185,28 @@ function checkpointSummaryFrom(...values) {
|
|
|
126
185
|
};
|
|
127
186
|
return Object.values(summary).some((value) => typeof value !== "undefined") ? summary : void 0;
|
|
128
187
|
}
|
|
188
|
+
function isProfileResult(result) {
|
|
189
|
+
const version = stringValue(result.version);
|
|
190
|
+
return version === "riddle-proof.profile-result.v1" || version === "riddle-proof.profile-compact-result.v1";
|
|
191
|
+
}
|
|
129
192
|
function summarizeRiddleProofPrComment(input) {
|
|
130
193
|
const runResponse = asRecord(input.runResponse);
|
|
131
194
|
const result = asRecord(input.result);
|
|
132
195
|
const proofResult = asRecord(runResponse.proofResult);
|
|
196
|
+
const profileResult = isProfileResult(result);
|
|
197
|
+
const profileRiddle = asRecord(result.riddle);
|
|
133
198
|
const preview = asRecord(runResponse.preview);
|
|
134
199
|
const resultRunCard = asRecord(result.run_card);
|
|
135
200
|
const stopCondition = asRecord(resultRunCard.stop_condition);
|
|
136
201
|
const resultDetails = asRecord(result.details);
|
|
137
202
|
const resultRaw = asRecord(result.raw);
|
|
138
203
|
const rawDetails = asRecord(resultRaw.details);
|
|
139
|
-
const
|
|
140
|
-
const
|
|
204
|
+
const profileChecks = profileResult ? profileCheckCounts(result) : void 0;
|
|
205
|
+
const artifacts = mergeArtifacts(
|
|
206
|
+
collectArtifacts(runResponse),
|
|
207
|
+
profileResult ? collectProfileArtifacts(result) : []
|
|
208
|
+
);
|
|
209
|
+
const pages = profileResult ? profilePageSummaries(result, profileChecks) : pageSummaries(result);
|
|
141
210
|
const checkSource = { ...result };
|
|
142
211
|
delete checkSource.ok;
|
|
143
212
|
const nestedChecks = summarizeExplicitChecks(checkSource);
|
|
@@ -160,10 +229,10 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
160
229
|
);
|
|
161
230
|
return {
|
|
162
231
|
ok,
|
|
163
|
-
status:
|
|
232
|
+
status: firstStringValue(proofResult.status, profileRiddle.status),
|
|
164
233
|
result_status: publicState.status,
|
|
165
|
-
job_id:
|
|
166
|
-
duration_ms: numberValue(proofResult.duration_ms),
|
|
234
|
+
job_id: firstStringValue(proofResult.job_id, profileRiddle.job_id),
|
|
235
|
+
duration_ms: numberValue(proofResult.duration_ms) ?? numberValue(profileRiddle.elapsed_ms),
|
|
167
236
|
proof_url: stringValue(runResponse.proofUrl),
|
|
168
237
|
preview_id: stringValue(preview.id),
|
|
169
238
|
preview_url: stringValue(preview.preview_url) || stringValue(preview.url),
|
|
@@ -178,8 +247,8 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
178
247
|
merge_recommendation: mergeRecommendation,
|
|
179
248
|
checkpoint_summary: checkpointSummary,
|
|
180
249
|
public_state: publicState,
|
|
181
|
-
passed_checks: nestedChecks.passed,
|
|
182
|
-
failed_checks: nestedChecks.failed,
|
|
250
|
+
passed_checks: profileChecks?.passed ?? nestedChecks.passed,
|
|
251
|
+
failed_checks: profileChecks?.failed ?? nestedChecks.failed,
|
|
183
252
|
pages,
|
|
184
253
|
artifacts,
|
|
185
254
|
primary_image: selectPrimaryImage(artifacts)
|
package/dist/cli/index.js
CHANGED
package/dist/cli.cjs
CHANGED
|
@@ -18034,6 +18034,36 @@ function collectArtifacts(runResponse) {
|
|
|
18034
18034
|
}
|
|
18035
18035
|
return artifacts;
|
|
18036
18036
|
}
|
|
18037
|
+
function collectProfileArtifacts(result) {
|
|
18038
|
+
const resultArtifacts = asRecord2(result.artifacts);
|
|
18039
|
+
const artifacts = [];
|
|
18040
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18041
|
+
for (const [index, item] of asArray(resultArtifacts.riddle_artifacts).entries()) {
|
|
18042
|
+
const artifact = asRecord2(item);
|
|
18043
|
+
const url = stringValue4(artifact.url);
|
|
18044
|
+
if (!url || seen.has(url)) continue;
|
|
18045
|
+
seen.add(url);
|
|
18046
|
+
const fallbackName = stringValue4(artifact.kind) || `artifact-${index + 1}`;
|
|
18047
|
+
const name = artifactDisplayName(artifact.name, fallbackName);
|
|
18048
|
+
artifacts.push({
|
|
18049
|
+
name,
|
|
18050
|
+
url,
|
|
18051
|
+
kind: artifactKind(name, url),
|
|
18052
|
+
size_bytes: numberValue3(artifact.size_bytes) ?? numberValue3(artifact.size)
|
|
18053
|
+
});
|
|
18054
|
+
}
|
|
18055
|
+
return artifacts;
|
|
18056
|
+
}
|
|
18057
|
+
function mergeArtifacts(...artifactLists) {
|
|
18058
|
+
const artifacts = [];
|
|
18059
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18060
|
+
for (const artifact of artifactLists.flat()) {
|
|
18061
|
+
if (seen.has(artifact.url)) continue;
|
|
18062
|
+
seen.add(artifact.url);
|
|
18063
|
+
artifacts.push(artifact);
|
|
18064
|
+
}
|
|
18065
|
+
return artifacts;
|
|
18066
|
+
}
|
|
18037
18067
|
function pageSummaries(result) {
|
|
18038
18068
|
const pages = [];
|
|
18039
18069
|
for (const page of asArray(result.pages)) {
|
|
@@ -18050,6 +18080,35 @@ function pageSummaries(result) {
|
|
|
18050
18080
|
}
|
|
18051
18081
|
return pages;
|
|
18052
18082
|
}
|
|
18083
|
+
function profileCheckCounts(result) {
|
|
18084
|
+
const checks = asArray(result.checks);
|
|
18085
|
+
if (checks.length) {
|
|
18086
|
+
let passed2 = 0;
|
|
18087
|
+
let failed2 = 0;
|
|
18088
|
+
for (const item of checks) {
|
|
18089
|
+
const status = stringValue4(asRecord2(item).status);
|
|
18090
|
+
if (status === "passed") passed2 += 1;
|
|
18091
|
+
if (status === "failed") failed2 += 1;
|
|
18092
|
+
}
|
|
18093
|
+
return { passed: passed2, failed: failed2 };
|
|
18094
|
+
}
|
|
18095
|
+
const checkCounts = asRecord2(result.check_counts);
|
|
18096
|
+
const passed = numberValue3(checkCounts.passed);
|
|
18097
|
+
const failed = numberValue3(checkCounts.failed);
|
|
18098
|
+
if (typeof passed === "number" || typeof failed === "number") {
|
|
18099
|
+
return { passed: passed ?? 0, failed: failed ?? 0 };
|
|
18100
|
+
}
|
|
18101
|
+
return void 0;
|
|
18102
|
+
}
|
|
18103
|
+
function profilePageSummaries(result, counts) {
|
|
18104
|
+
if (!counts) return [];
|
|
18105
|
+
const route = asRecord2(result.route);
|
|
18106
|
+
return [{
|
|
18107
|
+
route: firstStringValue2(route.observed, route.expected_path, route.requested, result.profile_name) || "profile",
|
|
18108
|
+
passed: counts.passed,
|
|
18109
|
+
failed: counts.failed
|
|
18110
|
+
}];
|
|
18111
|
+
}
|
|
18053
18112
|
function summarizeExplicitChecks(value) {
|
|
18054
18113
|
let passed = 0;
|
|
18055
18114
|
let failed = 0;
|
|
@@ -18101,18 +18160,28 @@ function checkpointSummaryFrom2(...values) {
|
|
|
18101
18160
|
};
|
|
18102
18161
|
return Object.values(summary).some((value) => typeof value !== "undefined") ? summary : void 0;
|
|
18103
18162
|
}
|
|
18163
|
+
function isProfileResult(result) {
|
|
18164
|
+
const version = stringValue4(result.version);
|
|
18165
|
+
return version === "riddle-proof.profile-result.v1" || version === "riddle-proof.profile-compact-result.v1";
|
|
18166
|
+
}
|
|
18104
18167
|
function summarizeRiddleProofPrComment(input) {
|
|
18105
18168
|
const runResponse = asRecord2(input.runResponse);
|
|
18106
18169
|
const result = asRecord2(input.result);
|
|
18107
18170
|
const proofResult = asRecord2(runResponse.proofResult);
|
|
18171
|
+
const profileResult = isProfileResult(result);
|
|
18172
|
+
const profileRiddle = asRecord2(result.riddle);
|
|
18108
18173
|
const preview = asRecord2(runResponse.preview);
|
|
18109
18174
|
const resultRunCard = asRecord2(result.run_card);
|
|
18110
18175
|
const stopCondition = asRecord2(resultRunCard.stop_condition);
|
|
18111
18176
|
const resultDetails = asRecord2(result.details);
|
|
18112
18177
|
const resultRaw = asRecord2(result.raw);
|
|
18113
18178
|
const rawDetails = asRecord2(resultRaw.details);
|
|
18114
|
-
const
|
|
18115
|
-
const
|
|
18179
|
+
const profileChecks = profileResult ? profileCheckCounts(result) : void 0;
|
|
18180
|
+
const artifacts = mergeArtifacts(
|
|
18181
|
+
collectArtifacts(runResponse),
|
|
18182
|
+
profileResult ? collectProfileArtifacts(result) : []
|
|
18183
|
+
);
|
|
18184
|
+
const pages = profileResult ? profilePageSummaries(result, profileChecks) : pageSummaries(result);
|
|
18116
18185
|
const checkSource = { ...result };
|
|
18117
18186
|
delete checkSource.ok;
|
|
18118
18187
|
const nestedChecks = summarizeExplicitChecks(checkSource);
|
|
@@ -18135,10 +18204,10 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
18135
18204
|
);
|
|
18136
18205
|
return {
|
|
18137
18206
|
ok,
|
|
18138
|
-
status:
|
|
18207
|
+
status: firstStringValue2(proofResult.status, profileRiddle.status),
|
|
18139
18208
|
result_status: publicState.status,
|
|
18140
|
-
job_id:
|
|
18141
|
-
duration_ms: numberValue3(proofResult.duration_ms),
|
|
18209
|
+
job_id: firstStringValue2(proofResult.job_id, profileRiddle.job_id),
|
|
18210
|
+
duration_ms: numberValue3(proofResult.duration_ms) ?? numberValue3(profileRiddle.elapsed_ms),
|
|
18142
18211
|
proof_url: stringValue4(runResponse.proofUrl),
|
|
18143
18212
|
preview_id: stringValue4(preview.id),
|
|
18144
18213
|
preview_url: stringValue4(preview.preview_url) || stringValue4(preview.url),
|
|
@@ -18153,8 +18222,8 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
18153
18222
|
merge_recommendation: mergeRecommendation,
|
|
18154
18223
|
checkpoint_summary: checkpointSummary,
|
|
18155
18224
|
public_state: publicState,
|
|
18156
|
-
passed_checks: nestedChecks.passed,
|
|
18157
|
-
failed_checks: nestedChecks.failed,
|
|
18225
|
+
passed_checks: profileChecks?.passed ?? nestedChecks.passed,
|
|
18226
|
+
failed_checks: profileChecks?.failed ?? nestedChecks.failed,
|
|
18158
18227
|
pages,
|
|
18159
18228
|
artifacts,
|
|
18160
18229
|
primary_image: selectPrimaryImage(artifacts)
|
|
@@ -23125,12 +23194,17 @@ async function main() {
|
|
|
23125
23194
|
}
|
|
23126
23195
|
if (command === "pr-comment") {
|
|
23127
23196
|
const proofDir = optionString(options, "proofDir") || optionString(options, "outputDir") || positional[1];
|
|
23128
|
-
const
|
|
23129
|
-
const
|
|
23197
|
+
const explicitRunResponsePath = optionString(options, "runResponse");
|
|
23198
|
+
const explicitResultJsonPath = optionString(options, "resultJson");
|
|
23199
|
+
const runResponsePath = explicitRunResponsePath || defaultProofDirJsonPath(proofDir, "riddle-run-response.json");
|
|
23200
|
+
const resultJsonPaths = explicitResultJsonPath ? [explicitResultJsonPath] : [
|
|
23201
|
+
defaultProofDirJsonPath(proofDir, "result.json"),
|
|
23202
|
+
defaultProofDirJsonPath(proofDir, "profile-result.json")
|
|
23203
|
+
];
|
|
23130
23204
|
const runResponse = readJsonFileIfExists(runResponsePath);
|
|
23131
|
-
const result = readJsonFileIfExists(
|
|
23205
|
+
const result = resultJsonPaths.map((candidate) => readJsonFileIfExists(candidate)).find(Boolean);
|
|
23132
23206
|
if (!runResponse && !result) {
|
|
23133
|
-
throw new Error("pr-comment requires --proof-dir with riddle-run-response.json
|
|
23207
|
+
throw new Error("pr-comment requires --proof-dir with riddle-run-response.json, result.json, or profile-result.json, or explicit --run-response/--result-json.");
|
|
23134
23208
|
}
|
|
23135
23209
|
const body = buildRiddleProofPrCommentMarkdown({
|
|
23136
23210
|
title: optionString(options, "title"),
|
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import "./chunk-
|
|
2
|
+
import "./chunk-AF3KFD4A.js";
|
|
3
3
|
import "./chunk-B2DP2LET.js";
|
|
4
|
-
import "./chunk-
|
|
4
|
+
import "./chunk-DTFSHRV5.js";
|
|
5
5
|
import "./chunk-5Y4V2IXI.js";
|
|
6
6
|
import "./chunk-EX7TO4I5.js";
|
|
7
7
|
import "./chunk-P4BHU5NM.js";
|
package/dist/index.cjs
CHANGED
|
@@ -20501,6 +20501,36 @@ function collectArtifacts(runResponse) {
|
|
|
20501
20501
|
}
|
|
20502
20502
|
return artifacts;
|
|
20503
20503
|
}
|
|
20504
|
+
function collectProfileArtifacts(result) {
|
|
20505
|
+
const resultArtifacts = asRecord2(result.artifacts);
|
|
20506
|
+
const artifacts = [];
|
|
20507
|
+
const seen = /* @__PURE__ */ new Set();
|
|
20508
|
+
for (const [index, item] of asArray(resultArtifacts.riddle_artifacts).entries()) {
|
|
20509
|
+
const artifact = asRecord2(item);
|
|
20510
|
+
const url = stringValue7(artifact.url);
|
|
20511
|
+
if (!url || seen.has(url)) continue;
|
|
20512
|
+
seen.add(url);
|
|
20513
|
+
const fallbackName = stringValue7(artifact.kind) || `artifact-${index + 1}`;
|
|
20514
|
+
const name = artifactDisplayName(artifact.name, fallbackName);
|
|
20515
|
+
artifacts.push({
|
|
20516
|
+
name,
|
|
20517
|
+
url,
|
|
20518
|
+
kind: artifactKind(name, url),
|
|
20519
|
+
size_bytes: numberValue5(artifact.size_bytes) ?? numberValue5(artifact.size)
|
|
20520
|
+
});
|
|
20521
|
+
}
|
|
20522
|
+
return artifacts;
|
|
20523
|
+
}
|
|
20524
|
+
function mergeArtifacts(...artifactLists) {
|
|
20525
|
+
const artifacts = [];
|
|
20526
|
+
const seen = /* @__PURE__ */ new Set();
|
|
20527
|
+
for (const artifact of artifactLists.flat()) {
|
|
20528
|
+
if (seen.has(artifact.url)) continue;
|
|
20529
|
+
seen.add(artifact.url);
|
|
20530
|
+
artifacts.push(artifact);
|
|
20531
|
+
}
|
|
20532
|
+
return artifacts;
|
|
20533
|
+
}
|
|
20504
20534
|
function pageSummaries(result) {
|
|
20505
20535
|
const pages = [];
|
|
20506
20536
|
for (const page of asArray(result.pages)) {
|
|
@@ -20517,6 +20547,35 @@ function pageSummaries(result) {
|
|
|
20517
20547
|
}
|
|
20518
20548
|
return pages;
|
|
20519
20549
|
}
|
|
20550
|
+
function profileCheckCounts(result) {
|
|
20551
|
+
const checks = asArray(result.checks);
|
|
20552
|
+
if (checks.length) {
|
|
20553
|
+
let passed2 = 0;
|
|
20554
|
+
let failed2 = 0;
|
|
20555
|
+
for (const item of checks) {
|
|
20556
|
+
const status = stringValue7(asRecord2(item).status);
|
|
20557
|
+
if (status === "passed") passed2 += 1;
|
|
20558
|
+
if (status === "failed") failed2 += 1;
|
|
20559
|
+
}
|
|
20560
|
+
return { passed: passed2, failed: failed2 };
|
|
20561
|
+
}
|
|
20562
|
+
const checkCounts = asRecord2(result.check_counts);
|
|
20563
|
+
const passed = numberValue5(checkCounts.passed);
|
|
20564
|
+
const failed = numberValue5(checkCounts.failed);
|
|
20565
|
+
if (typeof passed === "number" || typeof failed === "number") {
|
|
20566
|
+
return { passed: passed ?? 0, failed: failed ?? 0 };
|
|
20567
|
+
}
|
|
20568
|
+
return void 0;
|
|
20569
|
+
}
|
|
20570
|
+
function profilePageSummaries(result, counts) {
|
|
20571
|
+
if (!counts) return [];
|
|
20572
|
+
const route = asRecord2(result.route);
|
|
20573
|
+
return [{
|
|
20574
|
+
route: firstStringValue2(route.observed, route.expected_path, route.requested, result.profile_name) || "profile",
|
|
20575
|
+
passed: counts.passed,
|
|
20576
|
+
failed: counts.failed
|
|
20577
|
+
}];
|
|
20578
|
+
}
|
|
20520
20579
|
function summarizeExplicitChecks(value) {
|
|
20521
20580
|
let passed = 0;
|
|
20522
20581
|
let failed = 0;
|
|
@@ -20568,18 +20627,28 @@ function checkpointSummaryFrom2(...values) {
|
|
|
20568
20627
|
};
|
|
20569
20628
|
return Object.values(summary).some((value) => typeof value !== "undefined") ? summary : void 0;
|
|
20570
20629
|
}
|
|
20630
|
+
function isProfileResult(result) {
|
|
20631
|
+
const version = stringValue7(result.version);
|
|
20632
|
+
return version === "riddle-proof.profile-result.v1" || version === "riddle-proof.profile-compact-result.v1";
|
|
20633
|
+
}
|
|
20571
20634
|
function summarizeRiddleProofPrComment(input) {
|
|
20572
20635
|
const runResponse = asRecord2(input.runResponse);
|
|
20573
20636
|
const result = asRecord2(input.result);
|
|
20574
20637
|
const proofResult = asRecord2(runResponse.proofResult);
|
|
20638
|
+
const profileResult = isProfileResult(result);
|
|
20639
|
+
const profileRiddle = asRecord2(result.riddle);
|
|
20575
20640
|
const preview = asRecord2(runResponse.preview);
|
|
20576
20641
|
const resultRunCard = asRecord2(result.run_card);
|
|
20577
20642
|
const stopCondition = asRecord2(resultRunCard.stop_condition);
|
|
20578
20643
|
const resultDetails = asRecord2(result.details);
|
|
20579
20644
|
const resultRaw = asRecord2(result.raw);
|
|
20580
20645
|
const rawDetails = asRecord2(resultRaw.details);
|
|
20581
|
-
const
|
|
20582
|
-
const
|
|
20646
|
+
const profileChecks = profileResult ? profileCheckCounts(result) : void 0;
|
|
20647
|
+
const artifacts = mergeArtifacts(
|
|
20648
|
+
collectArtifacts(runResponse),
|
|
20649
|
+
profileResult ? collectProfileArtifacts(result) : []
|
|
20650
|
+
);
|
|
20651
|
+
const pages = profileResult ? profilePageSummaries(result, profileChecks) : pageSummaries(result);
|
|
20583
20652
|
const checkSource = { ...result };
|
|
20584
20653
|
delete checkSource.ok;
|
|
20585
20654
|
const nestedChecks = summarizeExplicitChecks(checkSource);
|
|
@@ -20602,10 +20671,10 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
20602
20671
|
);
|
|
20603
20672
|
return {
|
|
20604
20673
|
ok,
|
|
20605
|
-
status:
|
|
20674
|
+
status: firstStringValue2(proofResult.status, profileRiddle.status),
|
|
20606
20675
|
result_status: publicState.status,
|
|
20607
|
-
job_id:
|
|
20608
|
-
duration_ms: numberValue5(proofResult.duration_ms),
|
|
20676
|
+
job_id: firstStringValue2(proofResult.job_id, profileRiddle.job_id),
|
|
20677
|
+
duration_ms: numberValue5(proofResult.duration_ms) ?? numberValue5(profileRiddle.elapsed_ms),
|
|
20609
20678
|
proof_url: stringValue7(runResponse.proofUrl),
|
|
20610
20679
|
preview_id: stringValue7(preview.id),
|
|
20611
20680
|
preview_url: stringValue7(preview.preview_url) || stringValue7(preview.url),
|
|
@@ -20620,8 +20689,8 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
20620
20689
|
merge_recommendation: mergeRecommendation,
|
|
20621
20690
|
checkpoint_summary: checkpointSummary,
|
|
20622
20691
|
public_state: publicState,
|
|
20623
|
-
passed_checks: nestedChecks.passed,
|
|
20624
|
-
failed_checks: nestedChecks.failed,
|
|
20692
|
+
passed_checks: profileChecks?.passed ?? nestedChecks.passed,
|
|
20693
|
+
failed_checks: profileChecks?.failed ?? nestedChecks.failed,
|
|
20625
20694
|
pages,
|
|
20626
20695
|
artifacts,
|
|
20627
20696
|
primary_image: selectPrimaryImage(artifacts)
|
package/dist/index.js
CHANGED
|
@@ -60,7 +60,7 @@ import {
|
|
|
60
60
|
RIDDLE_PROOF_PR_COMMENT_MARKER,
|
|
61
61
|
buildRiddleProofPrCommentMarkdown,
|
|
62
62
|
summarizeRiddleProofPrComment
|
|
63
|
-
} from "./chunk-
|
|
63
|
+
} from "./chunk-DTFSHRV5.js";
|
|
64
64
|
import {
|
|
65
65
|
RIDDLE_PROOF_PROFILE_SUGGESTIONS_VERSION,
|
|
66
66
|
suggestRiddleProofProfileChecks
|
package/dist/pr-comment.cjs
CHANGED
|
@@ -242,6 +242,36 @@ function collectArtifacts(runResponse) {
|
|
|
242
242
|
}
|
|
243
243
|
return artifacts;
|
|
244
244
|
}
|
|
245
|
+
function collectProfileArtifacts(result) {
|
|
246
|
+
const resultArtifacts = asRecord2(result.artifacts);
|
|
247
|
+
const artifacts = [];
|
|
248
|
+
const seen = /* @__PURE__ */ new Set();
|
|
249
|
+
for (const [index, item] of asArray(resultArtifacts.riddle_artifacts).entries()) {
|
|
250
|
+
const artifact = asRecord2(item);
|
|
251
|
+
const url = stringValue2(artifact.url);
|
|
252
|
+
if (!url || seen.has(url)) continue;
|
|
253
|
+
seen.add(url);
|
|
254
|
+
const fallbackName = stringValue2(artifact.kind) || `artifact-${index + 1}`;
|
|
255
|
+
const name = artifactDisplayName(artifact.name, fallbackName);
|
|
256
|
+
artifacts.push({
|
|
257
|
+
name,
|
|
258
|
+
url,
|
|
259
|
+
kind: artifactKind(name, url),
|
|
260
|
+
size_bytes: numberValue2(artifact.size_bytes) ?? numberValue2(artifact.size)
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
return artifacts;
|
|
264
|
+
}
|
|
265
|
+
function mergeArtifacts(...artifactLists) {
|
|
266
|
+
const artifacts = [];
|
|
267
|
+
const seen = /* @__PURE__ */ new Set();
|
|
268
|
+
for (const artifact of artifactLists.flat()) {
|
|
269
|
+
if (seen.has(artifact.url)) continue;
|
|
270
|
+
seen.add(artifact.url);
|
|
271
|
+
artifacts.push(artifact);
|
|
272
|
+
}
|
|
273
|
+
return artifacts;
|
|
274
|
+
}
|
|
245
275
|
function pageSummaries(result) {
|
|
246
276
|
const pages = [];
|
|
247
277
|
for (const page of asArray(result.pages)) {
|
|
@@ -258,6 +288,35 @@ function pageSummaries(result) {
|
|
|
258
288
|
}
|
|
259
289
|
return pages;
|
|
260
290
|
}
|
|
291
|
+
function profileCheckCounts(result) {
|
|
292
|
+
const checks = asArray(result.checks);
|
|
293
|
+
if (checks.length) {
|
|
294
|
+
let passed2 = 0;
|
|
295
|
+
let failed2 = 0;
|
|
296
|
+
for (const item of checks) {
|
|
297
|
+
const status = stringValue2(asRecord2(item).status);
|
|
298
|
+
if (status === "passed") passed2 += 1;
|
|
299
|
+
if (status === "failed") failed2 += 1;
|
|
300
|
+
}
|
|
301
|
+
return { passed: passed2, failed: failed2 };
|
|
302
|
+
}
|
|
303
|
+
const checkCounts = asRecord2(result.check_counts);
|
|
304
|
+
const passed = numberValue2(checkCounts.passed);
|
|
305
|
+
const failed = numberValue2(checkCounts.failed);
|
|
306
|
+
if (typeof passed === "number" || typeof failed === "number") {
|
|
307
|
+
return { passed: passed ?? 0, failed: failed ?? 0 };
|
|
308
|
+
}
|
|
309
|
+
return void 0;
|
|
310
|
+
}
|
|
311
|
+
function profilePageSummaries(result, counts) {
|
|
312
|
+
if (!counts) return [];
|
|
313
|
+
const route = asRecord2(result.route);
|
|
314
|
+
return [{
|
|
315
|
+
route: firstStringValue2(route.observed, route.expected_path, route.requested, result.profile_name) || "profile",
|
|
316
|
+
passed: counts.passed,
|
|
317
|
+
failed: counts.failed
|
|
318
|
+
}];
|
|
319
|
+
}
|
|
261
320
|
function summarizeExplicitChecks(value) {
|
|
262
321
|
let passed = 0;
|
|
263
322
|
let failed = 0;
|
|
@@ -309,18 +368,28 @@ function checkpointSummaryFrom2(...values) {
|
|
|
309
368
|
};
|
|
310
369
|
return Object.values(summary).some((value) => typeof value !== "undefined") ? summary : void 0;
|
|
311
370
|
}
|
|
371
|
+
function isProfileResult(result) {
|
|
372
|
+
const version = stringValue2(result.version);
|
|
373
|
+
return version === "riddle-proof.profile-result.v1" || version === "riddle-proof.profile-compact-result.v1";
|
|
374
|
+
}
|
|
312
375
|
function summarizeRiddleProofPrComment(input) {
|
|
313
376
|
const runResponse = asRecord2(input.runResponse);
|
|
314
377
|
const result = asRecord2(input.result);
|
|
315
378
|
const proofResult = asRecord2(runResponse.proofResult);
|
|
379
|
+
const profileResult = isProfileResult(result);
|
|
380
|
+
const profileRiddle = asRecord2(result.riddle);
|
|
316
381
|
const preview = asRecord2(runResponse.preview);
|
|
317
382
|
const resultRunCard = asRecord2(result.run_card);
|
|
318
383
|
const stopCondition = asRecord2(resultRunCard.stop_condition);
|
|
319
384
|
const resultDetails = asRecord2(result.details);
|
|
320
385
|
const resultRaw = asRecord2(result.raw);
|
|
321
386
|
const rawDetails = asRecord2(resultRaw.details);
|
|
322
|
-
const
|
|
323
|
-
const
|
|
387
|
+
const profileChecks = profileResult ? profileCheckCounts(result) : void 0;
|
|
388
|
+
const artifacts = mergeArtifacts(
|
|
389
|
+
collectArtifacts(runResponse),
|
|
390
|
+
profileResult ? collectProfileArtifacts(result) : []
|
|
391
|
+
);
|
|
392
|
+
const pages = profileResult ? profilePageSummaries(result, profileChecks) : pageSummaries(result);
|
|
324
393
|
const checkSource = { ...result };
|
|
325
394
|
delete checkSource.ok;
|
|
326
395
|
const nestedChecks = summarizeExplicitChecks(checkSource);
|
|
@@ -343,10 +412,10 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
343
412
|
);
|
|
344
413
|
return {
|
|
345
414
|
ok,
|
|
346
|
-
status:
|
|
415
|
+
status: firstStringValue2(proofResult.status, profileRiddle.status),
|
|
347
416
|
result_status: publicState.status,
|
|
348
|
-
job_id:
|
|
349
|
-
duration_ms: numberValue2(proofResult.duration_ms),
|
|
417
|
+
job_id: firstStringValue2(proofResult.job_id, profileRiddle.job_id),
|
|
418
|
+
duration_ms: numberValue2(proofResult.duration_ms) ?? numberValue2(profileRiddle.elapsed_ms),
|
|
350
419
|
proof_url: stringValue2(runResponse.proofUrl),
|
|
351
420
|
preview_id: stringValue2(preview.id),
|
|
352
421
|
preview_url: stringValue2(preview.preview_url) || stringValue2(preview.url),
|
|
@@ -361,8 +430,8 @@ function summarizeRiddleProofPrComment(input) {
|
|
|
361
430
|
merge_recommendation: mergeRecommendation,
|
|
362
431
|
checkpoint_summary: checkpointSummary,
|
|
363
432
|
public_state: publicState,
|
|
364
|
-
passed_checks: nestedChecks.passed,
|
|
365
|
-
failed_checks: nestedChecks.failed,
|
|
433
|
+
passed_checks: profileChecks?.passed ?? nestedChecks.passed,
|
|
434
|
+
failed_checks: profileChecks?.failed ?? nestedChecks.failed,
|
|
366
435
|
pages,
|
|
367
436
|
artifacts,
|
|
368
437
|
primary_image: selectPrimaryImage(artifacts)
|
package/dist/pr-comment.js
CHANGED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Riddle Proof Story Matrices
|
|
2
|
+
|
|
3
|
+
`riddle-proof-bounded-loop.json` is a starter matrix for bounded dogfood loops.
|
|
4
|
+
It is meant to keep broad user-story sweeps evidence-backed and reviewable
|
|
5
|
+
without turning them into unbounded scaffolding.
|
|
6
|
+
|
|
7
|
+
Each story names:
|
|
8
|
+
|
|
9
|
+
- the Riddle Proof surface under test
|
|
10
|
+
- the expected behavior and expected verdict
|
|
11
|
+
- the runner to use
|
|
12
|
+
- the concrete receipts that must be saved
|
|
13
|
+
- likely failure classes
|
|
14
|
+
- the story status, such as candidate, ready, covered, blocked, or recurring
|
|
15
|
+
|
|
16
|
+
Use one batch at a time, save outputs under `artifacts/riddle-proof/<story-id>/`,
|
|
17
|
+
fix only high-signal failures, and rerun the same batch before expanding scope.
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "riddle-proof.story-matrix.v1",
|
|
3
|
+
"matrix_id": "riddle-proof-bounded-loop-2026-06",
|
|
4
|
+
"public_name": "Riddle Proof Bounded Dogfood Loop",
|
|
5
|
+
"description": "A small canonical matrix for exercising Riddle Proof as its own proving-ground product before scaling to broad generated user-story sweeps.",
|
|
6
|
+
"operating_rules": [
|
|
7
|
+
"Keep each pass bounded: choose one batch, run it, classify findings, fix only high-signal issues, then rerun the same batch.",
|
|
8
|
+
"A story is not passed until it records concrete evidence receipts such as command output, profile-result.json, proof.json, screenshots, PR comment body, or a formal/test artifact.",
|
|
9
|
+
"Do not count GitHub Actions failures caused by billing or unavailable runners as product regressions. Classify those as environment_blocked.",
|
|
10
|
+
"Prefer generic Riddle Proof surfaces over adapter-specific wrapper behavior unless the story explicitly names an adapter.",
|
|
11
|
+
"Negative-control stories are first-class: a correct product_regression or proof_insufficient result can be a pass for the framework."
|
|
12
|
+
],
|
|
13
|
+
"batch_policy": {
|
|
14
|
+
"default_batch_size": 8,
|
|
15
|
+
"max_batch_size": 20,
|
|
16
|
+
"rerun_rule": "After fixing a product or contract issue found by the batch, rerun every story in the same batch before advancing.",
|
|
17
|
+
"promotion_rule": "Promote a story from candidate to recurring only after it has at least one saved evidence packet and a stable expected verdict."
|
|
18
|
+
},
|
|
19
|
+
"status_values": [
|
|
20
|
+
"candidate",
|
|
21
|
+
"ready_to_run",
|
|
22
|
+
"covered_existing",
|
|
23
|
+
"blocked_external",
|
|
24
|
+
"needs_product_hook",
|
|
25
|
+
"recurring"
|
|
26
|
+
],
|
|
27
|
+
"failure_classes": [
|
|
28
|
+
"product_regression",
|
|
29
|
+
"proof_insufficient",
|
|
30
|
+
"environment_blocked",
|
|
31
|
+
"configuration_error",
|
|
32
|
+
"needs_human_review",
|
|
33
|
+
"contract_gap",
|
|
34
|
+
"doc_or_ux_gap"
|
|
35
|
+
],
|
|
36
|
+
"evidence_receipt_types": [
|
|
37
|
+
"profile-result.json",
|
|
38
|
+
"proof.json",
|
|
39
|
+
"summary.md",
|
|
40
|
+
"screenshot",
|
|
41
|
+
"console.json",
|
|
42
|
+
"dom-summary.json",
|
|
43
|
+
"regression-pack-result.json",
|
|
44
|
+
"pr-comment-body.md",
|
|
45
|
+
"preflight-json",
|
|
46
|
+
"formal-kernel-build",
|
|
47
|
+
"runtime-test-log",
|
|
48
|
+
"manual-review-note"
|
|
49
|
+
],
|
|
50
|
+
"stories": [
|
|
51
|
+
{
|
|
52
|
+
"id": "rp-story-profile-suggest-drafts-not-verdicts",
|
|
53
|
+
"batch": "authoring",
|
|
54
|
+
"surface": "profile-suggest",
|
|
55
|
+
"user_story": "As a proof author, I can ask Riddle Proof to suggest checks for a changed route without accidentally treating suggestions as proof.",
|
|
56
|
+
"expected_behavior": "The command emits a draft profile or JSON suggestion and clearly does not produce a proof verdict.",
|
|
57
|
+
"primary_runner": "local-cli",
|
|
58
|
+
"command": "riddle-proof-loop profile-suggest --route /proof --changed-files docs/proof.md --selectors main#main-content --format profile",
|
|
59
|
+
"expected_verdict": "draft_only",
|
|
60
|
+
"evidence_required": [
|
|
61
|
+
"generated profile includes route_loaded or selector checks",
|
|
62
|
+
"output contains no passed/product_regression/proof_insufficient verdict",
|
|
63
|
+
"operator records the saved draft path or stdout capture"
|
|
64
|
+
],
|
|
65
|
+
"receipt_types": ["manual-review-note"],
|
|
66
|
+
"failure_classes": ["doc_or_ux_gap", "contract_gap"],
|
|
67
|
+
"status": "candidate"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": "rp-story-hosted-happy-path-produces-artifacts",
|
|
71
|
+
"batch": "hosted-profile",
|
|
72
|
+
"surface": "run-profile hosted runner",
|
|
73
|
+
"user_story": "As a product maintainer, I can run a hosted profile against a live route and receive a passed verdict tied to screenshots and structured artifacts.",
|
|
74
|
+
"expected_behavior": "Hosted run-profile returns passed, writes profile-result.json, summary.md, proof.json, console.json, dom-summary.json, and records screenshot artifact names.",
|
|
75
|
+
"primary_runner": "hosted-riddle",
|
|
76
|
+
"command": "riddle-proof-loop run-profile --profile <route-smoke-profile> --url https://riddledc.com --runner riddle --output artifacts/riddle-proof/<story-id> --result-format compact-json",
|
|
77
|
+
"expected_verdict": "passed",
|
|
78
|
+
"evidence_required": [
|
|
79
|
+
"profile-result.json status is passed",
|
|
80
|
+
"riddle.job_id is present",
|
|
81
|
+
"proof_json and screenshot artifacts are listed"
|
|
82
|
+
],
|
|
83
|
+
"receipt_types": ["profile-result.json", "proof.json", "summary.md", "screenshot"],
|
|
84
|
+
"failure_classes": ["product_regression", "proof_insufficient", "environment_blocked"],
|
|
85
|
+
"existing_coverage": ["examples/regression-packs/oc-flow-regression.json: hosted-home-to-proof-route-change-pass"],
|
|
86
|
+
"status": "covered_existing"
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"id": "rp-story-hosted-negative-control-stays-regression",
|
|
90
|
+
"batch": "hosted-profile",
|
|
91
|
+
"surface": "run-profile hosted runner",
|
|
92
|
+
"user_story": "As a maintainer, I can run a negative-control profile and see a product_regression instead of a false pass.",
|
|
93
|
+
"expected_behavior": "A missing selector or missing query parameter produces product_regression with evidence, not passed and not a lifecycle crash.",
|
|
94
|
+
"primary_runner": "hosted-riddle",
|
|
95
|
+
"command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core false --hosted-riddle true --format markdown --output artifacts/riddle-proof/hosted-regression",
|
|
96
|
+
"expected_verdict": "product_regression",
|
|
97
|
+
"evidence_required": [
|
|
98
|
+
"negative case expected status is product_regression",
|
|
99
|
+
"profile-result.json includes the failed check",
|
|
100
|
+
"summary names the failed invariant without hiding artifacts"
|
|
101
|
+
],
|
|
102
|
+
"receipt_types": ["profile-result.json", "summary.md", "console.json"],
|
|
103
|
+
"failure_classes": ["contract_gap", "proof_insufficient", "environment_blocked"],
|
|
104
|
+
"existing_coverage": [
|
|
105
|
+
"examples/regression-packs/oc-flow-regression.json: hosted-pricing-query-hash-dropped-blocker",
|
|
106
|
+
"examples/regression-packs/oc-flow-regression.json: hosted-missing-selector-timeout-blocker"
|
|
107
|
+
],
|
|
108
|
+
"status": "covered_existing"
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "rp-story-cold-start-does-not-duplicate-too-soon",
|
|
112
|
+
"batch": "hosted-profile",
|
|
113
|
+
"surface": "hosted worker cold start",
|
|
114
|
+
"user_story": "As a hosted runner user, I can submit work while workers are scaled to zero without the client declaring the job stale before normal cold-start readiness.",
|
|
115
|
+
"expected_behavior": "The unsubmitted retry window is long enough for accepted scale-to-zero cold starts, and progress output explains worker-wake waiting.",
|
|
116
|
+
"primary_runner": "hosted-riddle",
|
|
117
|
+
"command": "riddle-proof-loop run-profile --profile <small-profile> --url <site> --runner riddle --output artifacts/riddle-proof/cold-start-smoke --result-format compact-json --progress-every-ms 5000",
|
|
118
|
+
"expected_verdict": "passed",
|
|
119
|
+
"evidence_required": [
|
|
120
|
+
"attempts budget corresponds to a five-minute unsubmitted timeout",
|
|
121
|
+
"progress output reports waiting_for_worker_submit before submission",
|
|
122
|
+
"profile-result.json passes or returns environment_blocked with the specific infra blocker"
|
|
123
|
+
],
|
|
124
|
+
"receipt_types": ["profile-result.json", "summary.md", "manual-review-note"],
|
|
125
|
+
"failure_classes": ["environment_blocked", "contract_gap"],
|
|
126
|
+
"existing_coverage": ["README cold-start retry documentation", "cli default timeout test"],
|
|
127
|
+
"status": "recurring"
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "rp-story-local-core-trust-boundary",
|
|
131
|
+
"batch": "local-core",
|
|
132
|
+
"surface": "runtime trust boundary",
|
|
133
|
+
"user_story": "As a framework maintainer, I can run the generic local core suite and prove route, query/hash, thrown-error, missing-selector, and no-diff cases without hosted infrastructure.",
|
|
134
|
+
"expected_behavior": "The local core regression suite passes all required cases and reports no forbidden terminal markers.",
|
|
135
|
+
"primary_runner": "local-core",
|
|
136
|
+
"command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle false --format compact-json --output artifacts/riddle-proof/local-core",
|
|
137
|
+
"expected_verdict": "passed",
|
|
138
|
+
"evidence_required": [
|
|
139
|
+
"regression-pack-result.json local_core.ok is true",
|
|
140
|
+
"missing_required_cases is empty",
|
|
141
|
+
"forbidden_terminal_markers_seen is empty"
|
|
142
|
+
],
|
|
143
|
+
"receipt_types": ["regression-pack-result.json", "runtime-test-log"],
|
|
144
|
+
"failure_classes": ["contract_gap", "configuration_error"],
|
|
145
|
+
"existing_coverage": ["runtime/tests/trust_boundary_regression.py"],
|
|
146
|
+
"status": "recurring"
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"id": "rp-story-pr-comment-renders-only-evidence-backed-status",
|
|
150
|
+
"batch": "reporting",
|
|
151
|
+
"surface": "pr-comment",
|
|
152
|
+
"user_story": "As a reviewer, I can get a PR comment whose badge, artifact links, and failed checks reflect the saved proof packet rather than raw guessed fields.",
|
|
153
|
+
"expected_behavior": "Dry-run PR comment generation renders status and artifacts from proof outputs and refuses missing required inputs.",
|
|
154
|
+
"primary_runner": "local-cli",
|
|
155
|
+
"command": "riddle-proof-loop pr-comment --proof-dir artifacts/riddle-proof/<story-id> --pr <number> --repo <owner/repo> --dry-run --body-file artifacts/riddle-proof/<story-id>/pr-comment-body.md",
|
|
156
|
+
"expected_verdict": "passed",
|
|
157
|
+
"evidence_required": [
|
|
158
|
+
"body file includes the same status as profile-result.json",
|
|
159
|
+
"required artifact links or local names are present",
|
|
160
|
+
"missing proof-dir inputs fail rather than rendering passed"
|
|
161
|
+
],
|
|
162
|
+
"receipt_types": ["pr-comment-body.md", "profile-result.json"],
|
|
163
|
+
"failure_classes": ["contract_gap", "doc_or_ux_gap"],
|
|
164
|
+
"existing_coverage": ["test.js pr-comment cases"],
|
|
165
|
+
"status": "ready_to_run"
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"id": "rp-story-profile-body-assertions-derive-from-real-artifact",
|
|
169
|
+
"batch": "artifact-publication",
|
|
170
|
+
"surface": "profile-body-assertions",
|
|
171
|
+
"user_story": "As a profile promoter, I can derive body assertions from a real artifact before making a public proof/profile claim.",
|
|
172
|
+
"expected_behavior": "The command emits snippets only when they are present in the artifact and fails when required fragments are missing.",
|
|
173
|
+
"primary_runner": "local-cli",
|
|
174
|
+
"command": "riddle-proof-loop profile-body-assertions --artifact <proof-json-or-url> --candidates-json '[\"product_regression\",\"passed\"]' --required-json '[\"product_regression\"]' --format body-contains",
|
|
175
|
+
"expected_verdict": "passed_or_failed_by_artifact",
|
|
176
|
+
"evidence_required": [
|
|
177
|
+
"required present fragment is emitted",
|
|
178
|
+
"required missing fragment exits nonzero",
|
|
179
|
+
"promoted profile records derived snippets instead of hand-authored guesses"
|
|
180
|
+
],
|
|
181
|
+
"receipt_types": ["preflight-json", "manual-review-note"],
|
|
182
|
+
"failure_classes": ["contract_gap", "proof_insufficient"],
|
|
183
|
+
"existing_coverage": ["README profile-body-assertions guidance", "test.js body assertion cases"],
|
|
184
|
+
"status": "ready_to_run"
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"id": "rp-story-http-preflight-catches-publication-mismatch",
|
|
188
|
+
"batch": "artifact-publication",
|
|
189
|
+
"surface": "profile-http-status-preflight",
|
|
190
|
+
"user_story": "As a public proof publisher, I can check artifact URLs and body fragments before spending a hosted browser run.",
|
|
191
|
+
"expected_behavior": "The preflight returns success only when status, content type, byte size, required body text, and forbidden body text all satisfy the profile.",
|
|
192
|
+
"primary_runner": "local-cli",
|
|
193
|
+
"command": "riddle-proof-loop profile-http-status-preflight --profile <public-promotion-profile> --url <preview-url> --format json",
|
|
194
|
+
"expected_verdict": "passed_or_product_regression",
|
|
195
|
+
"evidence_required": [
|
|
196
|
+
"preflight output lists every checked URL",
|
|
197
|
+
"missing required body text fails",
|
|
198
|
+
"forbidden escaped/raw artifact text fails"
|
|
199
|
+
],
|
|
200
|
+
"receipt_types": ["preflight-json"],
|
|
201
|
+
"failure_classes": ["product_regression", "proof_insufficient", "configuration_error"],
|
|
202
|
+
"existing_coverage": ["test.js http preflight cases"],
|
|
203
|
+
"status": "ready_to_run"
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"id": "rp-story-run-profile-recover-preserves-artifacts",
|
|
207
|
+
"batch": "hosted-profile",
|
|
208
|
+
"surface": "run-profile recover",
|
|
209
|
+
"user_story": "As a user whose local process was interrupted, I can recover a hosted job and still get the proof artifacts and summary.",
|
|
210
|
+
"expected_behavior": "Recover fetches the hosted job artifacts, writes the same profile-result/summary/proof files, and preserves the original job id.",
|
|
211
|
+
"primary_runner": "hosted-riddle",
|
|
212
|
+
"command": "riddle-proof-loop run-profile recover --profile <profile> --url <base-url> --job <job-id> --output artifacts/riddle-proof/<story-id>-recovered --result-format compact-json",
|
|
213
|
+
"expected_verdict": "passed_or_original_verdict",
|
|
214
|
+
"evidence_required": [
|
|
215
|
+
"recovered profile-result.json includes the requested job id",
|
|
216
|
+
"summary.md is written",
|
|
217
|
+
"proof.json and screenshots are recovered or explicitly reported missing"
|
|
218
|
+
],
|
|
219
|
+
"receipt_types": ["profile-result.json", "summary.md", "proof.json", "screenshot"],
|
|
220
|
+
"failure_classes": ["environment_blocked", "proof_insufficient", "contract_gap"],
|
|
221
|
+
"status": "candidate"
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
"id": "rp-story-viewport-aggregate-does-not-hide-child-failure",
|
|
225
|
+
"batch": "hosted-profile",
|
|
226
|
+
"surface": "run-profile aggregate",
|
|
227
|
+
"user_story": "As a maintainer running viewport matrices, I can aggregate child runs without losing the failing viewport or overstating total latency.",
|
|
228
|
+
"expected_behavior": "Aggregate returns the worst required child verdict, lists child outputs, and separates summed poll time from max child latency.",
|
|
229
|
+
"primary_runner": "local-cli",
|
|
230
|
+
"command": "riddle-proof-loop run-profile aggregate --profile <matrix-profile> --url <base-url> --input-dir artifacts/riddle-proof/<matrix-dir> --output artifacts/riddle-proof/<matrix-dir>",
|
|
231
|
+
"expected_verdict": "worst_child_verdict",
|
|
232
|
+
"evidence_required": [
|
|
233
|
+
"aggregate profile-result.json names all child viewports",
|
|
234
|
+
"any failed child remains visible in checks",
|
|
235
|
+
"summary reports child timing without implying a single hosted job"
|
|
236
|
+
],
|
|
237
|
+
"receipt_types": ["profile-result.json", "summary.md"],
|
|
238
|
+
"failure_classes": ["contract_gap", "proof_insufficient"],
|
|
239
|
+
"existing_coverage": ["test.js aggregate cases"],
|
|
240
|
+
"status": "ready_to_run"
|
|
241
|
+
},
|
|
242
|
+
{
|
|
243
|
+
"id": "rp-story-formal-kernel-matches-runtime-contract",
|
|
244
|
+
"batch": "formal-contract",
|
|
245
|
+
"surface": "formal/riddle-proof-kernel",
|
|
246
|
+
"user_story": "As a framework maintainer, I can prove the result/status contract that runtime tests depend on, without putting Lean in the evidence-collection hot path.",
|
|
247
|
+
"expected_behavior": "The formal kernel builds, and conformance tests map the same runtime cases into executable checks.",
|
|
248
|
+
"primary_runner": "lean-kernel",
|
|
249
|
+
"command": "cd formal/riddle-proof-kernel && lake build && cd ../.. && node packages/riddle-proof/formal-conformance.test.js",
|
|
250
|
+
"expected_verdict": "passed",
|
|
251
|
+
"evidence_required": [
|
|
252
|
+
"lake build exits zero",
|
|
253
|
+
"formal-conformance.test.js exits zero",
|
|
254
|
+
"new runtime status cases are either mirrored in Lean or explicitly classified as out of formal scope"
|
|
255
|
+
],
|
|
256
|
+
"receipt_types": ["formal-kernel-build", "runtime-test-log"],
|
|
257
|
+
"failure_classes": ["contract_gap", "configuration_error"],
|
|
258
|
+
"existing_coverage": ["formal/riddle-proof-kernel", "formal-conformance.test.js"],
|
|
259
|
+
"status": "recurring"
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
"id": "rp-story-agent-loop-checkpoint-stale-response",
|
|
263
|
+
"batch": "agent-loop",
|
|
264
|
+
"surface": "checkpoint/respond/status",
|
|
265
|
+
"user_story": "As a host integration, I can ignore a stale checkpoint response after the run has already reached a terminal status.",
|
|
266
|
+
"expected_behavior": "A late response is recorded as ignored and does not reopen the run or start background resume.",
|
|
267
|
+
"primary_runner": "local-core",
|
|
268
|
+
"command": "riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle false --format json --output artifacts/riddle-proof/checkpoint-stale",
|
|
269
|
+
"expected_verdict": "passed",
|
|
270
|
+
"evidence_required": [
|
|
271
|
+
"late-stale-checkpoint-ignored case is present",
|
|
272
|
+
"ignored_checkpoint_response is true",
|
|
273
|
+
"background_resume_started is false"
|
|
274
|
+
],
|
|
275
|
+
"receipt_types": ["regression-pack-result.json", "runtime-test-log"],
|
|
276
|
+
"failure_classes": ["contract_gap"],
|
|
277
|
+
"existing_coverage": ["examples/regression-packs/oc-flow-regression.json: late-stale-checkpoint-ignored"],
|
|
278
|
+
"status": "covered_existing"
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
"id": "rp-story-hosted-proof-view-uses-contract",
|
|
282
|
+
"batch": "hosted-proof-view",
|
|
283
|
+
"surface": "hosted proof renderer",
|
|
284
|
+
"user_story": "As a viewer of a hosted proof page, I see the contract-derived status, artifacts, and missing-evidence explanation rather than raw backend fields.",
|
|
285
|
+
"expected_behavior": "Hosted proof views and any agent-facing summaries import/use the same status/artifact contract as PR comments.",
|
|
286
|
+
"primary_runner": "manual-review",
|
|
287
|
+
"command": "Inspect hosted proof renderer imports and run a hosted proof view smoke with a passed and a negative-control packet.",
|
|
288
|
+
"expected_verdict": "needs_human_review",
|
|
289
|
+
"evidence_required": [
|
|
290
|
+
"renderer uses shared contract code or has a documented gap",
|
|
291
|
+
"passed packet renders passed with artifacts",
|
|
292
|
+
"negative-control packet renders product_regression or proof_insufficient, never passed"
|
|
293
|
+
],
|
|
294
|
+
"receipt_types": ["manual-review-note", "screenshot", "profile-result.json"],
|
|
295
|
+
"failure_classes": ["contract_gap", "doc_or_ux_gap"],
|
|
296
|
+
"status": "needs_product_hook"
|
|
297
|
+
}
|
|
298
|
+
]
|
|
299
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@riddledc/riddle-proof",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.68",
|
|
4
4
|
"description": "Reusable Riddle Proof contracts and helpers for evidence-backed agent changes.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "RiddleDC",
|
|
@@ -247,6 +247,6 @@
|
|
|
247
247
|
"build": "npm --workspace @riddledc/riddle-proof-app-contract run build --if-present && tsup src/index.ts src/types.ts src/result.ts src/state.ts src/checkpoint.ts src/run-card.ts src/public-state.ts src/runner.ts src/engine-harness.ts src/codex-exec-agent.ts src/local-agent.ts src/cli.ts src/cli/index.ts src/diagnostics.ts src/proof-session.ts src/playability.ts src/basic-gameplay.ts src/profile.ts src/profile/index.ts src/profile-suggestions.ts src/pr-comment.ts src/openclaw.ts src/proof-run-core.ts src/proof-run-engine.ts src/riddle-client.ts src/runtime/riddle-client.ts src/spec/index.ts src/spec/types.ts src/spec/result.ts src/spec/state.ts src/spec/checkpoint.ts src/spec/run-card.ts src/spec/public-state.ts src/runtime/index.ts src/app-contract/index.ts src/advanced/index.ts src/advanced/runner.ts src/advanced/engine-harness.ts src/advanced/proof-run-core.ts src/advanced/proof-run-engine.ts src/adapters/openclaw.ts src/adapters/local-agent.ts src/adapters/codex-exec-agent.ts src/adapters/codex.ts --format cjs,esm --dts --out-dir dist --clean",
|
|
248
248
|
"clean": "rm -rf dist",
|
|
249
249
|
"lint": "echo 'lint: (not configured)'",
|
|
250
|
-
"test": "npm run build && node test.js && node proof-run.test.js && node formal-conformance.test.js && node trust-boundary.test.js && node regression-packs.test.js && python3 runtime/tests/trust_boundary_regression.py && python3 runtime/tests/ship_artifact_publication.py && (python3 runtime/tests/recon_verify_smoke.py >/tmp/riddle-proof-recon-verify-smoke.json || (tail -120 /tmp/riddle-proof-recon-verify-smoke.json; exit 1))"
|
|
250
|
+
"test": "npm run build && node test.js && node proof-run.test.js && node formal-conformance.test.js && node trust-boundary.test.js && node regression-packs.test.js && node story-matrix.test.js && python3 runtime/tests/trust_boundary_regression.py && python3 runtime/tests/ship_artifact_publication.py && (python3 runtime/tests/recon_verify_smoke.py >/tmp/riddle-proof-recon-verify-smoke.json || (tail -120 /tmp/riddle-proof-recon-verify-smoke.json; exit 1))"
|
|
251
251
|
}
|
|
252
252
|
}
|