@riddledc/riddle-proof 0.8.33 → 0.8.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-U73JPBZW.js → chunk-UGHN77PS.js} +286 -6
- package/dist/cli/index.js +1 -1
- package/dist/cli.cjs +286 -6
- package/dist/cli.js +1 -1
- package/examples/regression-packs/oc-flow-regression.json +198 -0
- package/package.json +1 -1
- package/runtime/lib/ship.py +126 -1
- package/runtime/tests/ship_artifact_publication.py +13 -5
|
@@ -65,6 +65,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
65
65
|
"format",
|
|
66
66
|
"framework",
|
|
67
67
|
"help",
|
|
68
|
+
"hostedRiddle",
|
|
68
69
|
"image",
|
|
69
70
|
"input",
|
|
70
71
|
"inputDir",
|
|
@@ -136,7 +137,7 @@ function usage() {
|
|
|
136
137
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
137
138
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
138
139
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
139
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
141
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
141
142
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
142
143
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -447,6 +448,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
447
448
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
448
449
|
};
|
|
449
450
|
}
|
|
451
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
452
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
453
|
+
}
|
|
454
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
455
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
456
|
+
}
|
|
457
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
458
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
459
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
460
|
+
}
|
|
461
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
462
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
463
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
464
|
+
}
|
|
465
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
466
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
467
|
+
return cliString(expect.message_contains);
|
|
468
|
+
}
|
|
469
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
470
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
471
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
472
|
+
return {
|
|
473
|
+
requested: false,
|
|
474
|
+
configured: cases.length > 0,
|
|
475
|
+
ok: true,
|
|
476
|
+
runner: cliString(suite.runner) || "riddle",
|
|
477
|
+
target: regressionPackRecord(suite.target),
|
|
478
|
+
case_count: cases.length,
|
|
479
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
483
|
+
return outputDir ? path.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
484
|
+
}
|
|
485
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
486
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
487
|
+
return {
|
|
488
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
489
|
+
intent: cliString(testCase.intent) || null,
|
|
490
|
+
ok,
|
|
491
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
492
|
+
expected_message_contains: expectedMessage || null,
|
|
493
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
494
|
+
status: result.status,
|
|
495
|
+
profile_name: result.profile_name,
|
|
496
|
+
summary: result.summary,
|
|
497
|
+
route: result.route,
|
|
498
|
+
riddle: result.riddle,
|
|
499
|
+
artifacts: result.artifacts,
|
|
500
|
+
environment_blocker: result.environment_blocker,
|
|
501
|
+
error: result.error
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
505
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
506
|
+
return {
|
|
507
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
508
|
+
intent: cliString(testCase.intent) || null,
|
|
509
|
+
ok: false,
|
|
510
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
511
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
512
|
+
status: "environment_blocked",
|
|
513
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
514
|
+
summary: error,
|
|
515
|
+
environment_blocker: environmentBlocker,
|
|
516
|
+
error
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
520
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
521
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
522
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
523
|
+
let balance;
|
|
524
|
+
try {
|
|
525
|
+
balance = await client.getBalance();
|
|
526
|
+
} catch (error) {
|
|
527
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
528
|
+
return {
|
|
529
|
+
error: message,
|
|
530
|
+
environment_blocker: {
|
|
531
|
+
source: "riddle_api",
|
|
532
|
+
endpoint: "/v1/balance",
|
|
533
|
+
reason: "balance_preflight_failed",
|
|
534
|
+
balance_preflight: true,
|
|
535
|
+
job_count: cases.length,
|
|
536
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
537
|
+
required_seconds: requiredSeconds,
|
|
538
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
539
|
+
...apiKeySourceBlockerMetadata(client)
|
|
540
|
+
}
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
544
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
545
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
546
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
547
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
548
|
+
return {
|
|
549
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
550
|
+
environment_blocker: {
|
|
551
|
+
source: "riddle_api",
|
|
552
|
+
endpoint: "/v1/balance",
|
|
553
|
+
reason: "insufficient_balance",
|
|
554
|
+
error: "Insufficient available balance",
|
|
555
|
+
balance_preflight: true,
|
|
556
|
+
job_count: cases.length,
|
|
557
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
558
|
+
required_seconds: requiredSeconds,
|
|
559
|
+
available_seconds: availableSeconds,
|
|
560
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
561
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
562
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
563
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
564
|
+
...apiKeySourceBlockerMetadata(client)
|
|
565
|
+
}
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
569
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
570
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
571
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
572
|
+
const target = regressionPackRecord(suite.target);
|
|
573
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
574
|
+
const outputDir = profileOutputDirOption(options);
|
|
575
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
576
|
+
const results = [];
|
|
577
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
578
|
+
if (suiteBlocker) {
|
|
579
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
580
|
+
return {
|
|
581
|
+
requested: true,
|
|
582
|
+
configured: cases.length > 0,
|
|
583
|
+
ok: false,
|
|
584
|
+
runner,
|
|
585
|
+
target,
|
|
586
|
+
started_at: startedAt,
|
|
587
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
588
|
+
case_count: cases.length,
|
|
589
|
+
passed_case_count: 0,
|
|
590
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
591
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
592
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
593
|
+
error: suiteBlocker.error,
|
|
594
|
+
cases: blockedCases
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
for (const testCase of cases) {
|
|
598
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
599
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
600
|
+
if (!Object.keys(rawProfile).length) {
|
|
601
|
+
results.push({
|
|
602
|
+
id: caseId,
|
|
603
|
+
intent: cliString(testCase.intent) || null,
|
|
604
|
+
ok: false,
|
|
605
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
606
|
+
status: "configuration_error",
|
|
607
|
+
error: "hosted_riddle_suite case requires profile."
|
|
608
|
+
});
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
let result;
|
|
612
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
613
|
+
const caseOptions = {
|
|
614
|
+
...options,
|
|
615
|
+
runner,
|
|
616
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
617
|
+
};
|
|
618
|
+
try {
|
|
619
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
620
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
621
|
+
options
|
|
622
|
+
);
|
|
623
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
624
|
+
writeProfileOutput(caseOutputDir, result);
|
|
625
|
+
} catch (error) {
|
|
626
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
627
|
+
results.push({
|
|
628
|
+
id: caseId,
|
|
629
|
+
intent: cliString(testCase.intent) || null,
|
|
630
|
+
ok: false,
|
|
631
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
632
|
+
status: "configuration_error",
|
|
633
|
+
error: message,
|
|
634
|
+
output_dir: caseOutputDir || null
|
|
635
|
+
});
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
639
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
640
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
641
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
642
|
+
const ok = statusOk && messageOk;
|
|
643
|
+
results.push({
|
|
644
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
645
|
+
output_dir: caseOutputDir || null
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
649
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
650
|
+
return {
|
|
651
|
+
requested: true,
|
|
652
|
+
configured: cases.length > 0,
|
|
653
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
654
|
+
runner,
|
|
655
|
+
target,
|
|
656
|
+
started_at: startedAt,
|
|
657
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
658
|
+
case_count: cases.length,
|
|
659
|
+
passed_case_count: results.length - failedCases.length,
|
|
660
|
+
failed_cases: failedCases,
|
|
661
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
662
|
+
cases: results
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
666
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
667
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
668
|
+
const target = regressionPackRecord(suite.target);
|
|
669
|
+
const lines = [
|
|
670
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
671
|
+
"",
|
|
672
|
+
"Command:",
|
|
673
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
674
|
+
"",
|
|
675
|
+
"Target:",
|
|
676
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
677
|
+
"",
|
|
678
|
+
"Cases:",
|
|
679
|
+
...cases.map((testCase, index) => {
|
|
680
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
681
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
682
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
683
|
+
}),
|
|
684
|
+
"",
|
|
685
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
686
|
+
].filter((line) => line !== "");
|
|
687
|
+
return lines.join("\n");
|
|
688
|
+
}
|
|
450
689
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
451
690
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
452
691
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -468,6 +707,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
468
707
|
"- Run cases serially, not as one broad parallel batch.",
|
|
469
708
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
470
709
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
710
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
471
711
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
472
712
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
473
713
|
"",
|
|
@@ -480,12 +720,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
480
720
|
].join("\n");
|
|
481
721
|
}),
|
|
482
722
|
"",
|
|
483
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
723
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
484
724
|
].filter((line) => line !== "");
|
|
485
725
|
return lines.join("\n");
|
|
486
726
|
}
|
|
487
727
|
function compactRegressionPackRunResult(result) {
|
|
488
728
|
const localCore = regressionPackRecord(result.local_core);
|
|
729
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
489
730
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
490
731
|
return {
|
|
491
732
|
version: result.version,
|
|
@@ -500,12 +741,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
500
741
|
failed_cases: localCore.failed_cases,
|
|
501
742
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
502
743
|
},
|
|
744
|
+
hosted_riddle: {
|
|
745
|
+
requested: hostedRiddle.requested,
|
|
746
|
+
configured: hostedRiddle.configured,
|
|
747
|
+
ok: hostedRiddle.ok,
|
|
748
|
+
runner: hostedRiddle.runner,
|
|
749
|
+
case_count: hostedRiddle.case_count,
|
|
750
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
751
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
752
|
+
},
|
|
503
753
|
openclaw_live_case_count: openClaw.case_count,
|
|
504
754
|
output_dir: result.output_dir
|
|
505
755
|
};
|
|
506
756
|
}
|
|
507
757
|
function regressionPackRunMarkdown(result) {
|
|
508
758
|
const localCore = regressionPackRecord(result.local_core);
|
|
759
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
509
760
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
510
761
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
511
762
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -525,6 +776,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
525
776
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
526
777
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
527
778
|
"",
|
|
779
|
+
"## Hosted Riddle",
|
|
780
|
+
"",
|
|
781
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
782
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
783
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
784
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
785
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
786
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
787
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
788
|
+
"",
|
|
528
789
|
"## Runtime Gate",
|
|
529
790
|
"",
|
|
530
791
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -535,6 +796,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
535
796
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
536
797
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
537
798
|
"",
|
|
799
|
+
"## Hosted Riddle Handoff",
|
|
800
|
+
"",
|
|
801
|
+
"```text",
|
|
802
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
803
|
+
"```",
|
|
804
|
+
"",
|
|
538
805
|
"## OC Handoff Prompt",
|
|
539
806
|
"",
|
|
540
807
|
"```text",
|
|
@@ -551,22 +818,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
551
818
|
writeFileSync(path.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
552
819
|
`);
|
|
553
820
|
writeFileSync(path.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
821
|
+
writeFileSync(path.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
822
|
+
`);
|
|
554
823
|
writeFileSync(path.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
555
824
|
`);
|
|
556
825
|
}
|
|
557
|
-
function runRegressionPackForCli(options) {
|
|
826
|
+
async function runRegressionPackForCli(options) {
|
|
558
827
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
559
828
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
829
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
560
830
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
831
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
561
832
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
562
833
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
563
834
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
835
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
564
836
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
565
|
-
const
|
|
837
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
838
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
566
839
|
const result = {
|
|
567
840
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
568
841
|
ok,
|
|
569
842
|
local_core_validated: localCoreValidated,
|
|
843
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
570
844
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
571
845
|
pack_path: filePath,
|
|
572
846
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -576,13 +850,19 @@ function runRegressionPackForCli(options) {
|
|
|
576
850
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
577
851
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
578
852
|
local_core: localCore,
|
|
853
|
+
hosted_riddle: hostedRiddle,
|
|
579
854
|
openclaw_live_suite: {
|
|
580
855
|
target: regressionPackRecord(liveSuite.target),
|
|
581
856
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
582
857
|
case_count: liveCases.length,
|
|
583
858
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
584
859
|
},
|
|
585
|
-
|
|
860
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
861
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
862
|
+
localCoreOk: localCoreValidated,
|
|
863
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
864
|
+
hostedRiddleRequested
|
|
865
|
+
}),
|
|
586
866
|
output_dir: profileOutputDirOption(options) || null
|
|
587
867
|
};
|
|
588
868
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -4269,7 +4549,7 @@ async function main() {
|
|
|
4269
4549
|
if (command === "regression-pack") {
|
|
4270
4550
|
const action = positional[1] || "run";
|
|
4271
4551
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
4272
|
-
const result = runRegressionPackForCli(options);
|
|
4552
|
+
const result = await runRegressionPackForCli(options);
|
|
4273
4553
|
writeRegressionPackRunResult(result, options);
|
|
4274
4554
|
process.exitCode = result.ok ? 0 : 1;
|
|
4275
4555
|
return;
|
package/dist/cli/index.js
CHANGED
package/dist/cli.cjs
CHANGED
|
@@ -17157,6 +17157,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
17157
17157
|
"format",
|
|
17158
17158
|
"framework",
|
|
17159
17159
|
"help",
|
|
17160
|
+
"hostedRiddle",
|
|
17160
17161
|
"image",
|
|
17161
17162
|
"input",
|
|
17162
17163
|
"inputDir",
|
|
@@ -17228,7 +17229,7 @@ function usage() {
|
|
|
17228
17229
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
17229
17230
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17230
17231
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17231
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17232
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17232
17233
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
17233
17234
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
17234
17235
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -17539,6 +17540,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
17539
17540
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
17540
17541
|
};
|
|
17541
17542
|
}
|
|
17543
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
17544
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
17545
|
+
}
|
|
17546
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
17547
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
17548
|
+
}
|
|
17549
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
17550
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17551
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
17552
|
+
}
|
|
17553
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
17554
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17555
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
17556
|
+
}
|
|
17557
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
17558
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17559
|
+
return cliString(expect.message_contains);
|
|
17560
|
+
}
|
|
17561
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
17562
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17563
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17564
|
+
return {
|
|
17565
|
+
requested: false,
|
|
17566
|
+
configured: cases.length > 0,
|
|
17567
|
+
ok: true,
|
|
17568
|
+
runner: cliString(suite.runner) || "riddle",
|
|
17569
|
+
target: regressionPackRecord(suite.target),
|
|
17570
|
+
case_count: cases.length,
|
|
17571
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
17572
|
+
};
|
|
17573
|
+
}
|
|
17574
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
17575
|
+
return outputDir ? import_node_path6.default.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
17576
|
+
}
|
|
17577
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
17578
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17579
|
+
return {
|
|
17580
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
17581
|
+
intent: cliString(testCase.intent) || null,
|
|
17582
|
+
ok,
|
|
17583
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17584
|
+
expected_message_contains: expectedMessage || null,
|
|
17585
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
17586
|
+
status: result.status,
|
|
17587
|
+
profile_name: result.profile_name,
|
|
17588
|
+
summary: result.summary,
|
|
17589
|
+
route: result.route,
|
|
17590
|
+
riddle: result.riddle,
|
|
17591
|
+
artifacts: result.artifacts,
|
|
17592
|
+
environment_blocker: result.environment_blocker,
|
|
17593
|
+
error: result.error
|
|
17594
|
+
};
|
|
17595
|
+
}
|
|
17596
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
17597
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17598
|
+
return {
|
|
17599
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
17600
|
+
intent: cliString(testCase.intent) || null,
|
|
17601
|
+
ok: false,
|
|
17602
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17603
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
17604
|
+
status: "environment_blocked",
|
|
17605
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
17606
|
+
summary: error,
|
|
17607
|
+
environment_blocker: environmentBlocker,
|
|
17608
|
+
error
|
|
17609
|
+
};
|
|
17610
|
+
}
|
|
17611
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
17612
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
17613
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
17614
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
17615
|
+
let balance;
|
|
17616
|
+
try {
|
|
17617
|
+
balance = await client.getBalance();
|
|
17618
|
+
} catch (error) {
|
|
17619
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17620
|
+
return {
|
|
17621
|
+
error: message,
|
|
17622
|
+
environment_blocker: {
|
|
17623
|
+
source: "riddle_api",
|
|
17624
|
+
endpoint: "/v1/balance",
|
|
17625
|
+
reason: "balance_preflight_failed",
|
|
17626
|
+
balance_preflight: true,
|
|
17627
|
+
job_count: cases.length,
|
|
17628
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17629
|
+
required_seconds: requiredSeconds,
|
|
17630
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
17631
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17632
|
+
}
|
|
17633
|
+
};
|
|
17634
|
+
}
|
|
17635
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
17636
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
17637
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
17638
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
17639
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
17640
|
+
return {
|
|
17641
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
17642
|
+
environment_blocker: {
|
|
17643
|
+
source: "riddle_api",
|
|
17644
|
+
endpoint: "/v1/balance",
|
|
17645
|
+
reason: "insufficient_balance",
|
|
17646
|
+
error: "Insufficient available balance",
|
|
17647
|
+
balance_preflight: true,
|
|
17648
|
+
job_count: cases.length,
|
|
17649
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17650
|
+
required_seconds: requiredSeconds,
|
|
17651
|
+
available_seconds: availableSeconds,
|
|
17652
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
17653
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
17654
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
17655
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
17656
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17657
|
+
}
|
|
17658
|
+
};
|
|
17659
|
+
}
|
|
17660
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
17661
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17662
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17663
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
17664
|
+
const target = regressionPackRecord(suite.target);
|
|
17665
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
17666
|
+
const outputDir = profileOutputDirOption(options);
|
|
17667
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17668
|
+
const results = [];
|
|
17669
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
17670
|
+
if (suiteBlocker) {
|
|
17671
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
17672
|
+
return {
|
|
17673
|
+
requested: true,
|
|
17674
|
+
configured: cases.length > 0,
|
|
17675
|
+
ok: false,
|
|
17676
|
+
runner,
|
|
17677
|
+
target,
|
|
17678
|
+
started_at: startedAt,
|
|
17679
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17680
|
+
case_count: cases.length,
|
|
17681
|
+
passed_case_count: 0,
|
|
17682
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17683
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17684
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
17685
|
+
error: suiteBlocker.error,
|
|
17686
|
+
cases: blockedCases
|
|
17687
|
+
};
|
|
17688
|
+
}
|
|
17689
|
+
for (const testCase of cases) {
|
|
17690
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
17691
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17692
|
+
if (!Object.keys(rawProfile).length) {
|
|
17693
|
+
results.push({
|
|
17694
|
+
id: caseId,
|
|
17695
|
+
intent: cliString(testCase.intent) || null,
|
|
17696
|
+
ok: false,
|
|
17697
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17698
|
+
status: "configuration_error",
|
|
17699
|
+
error: "hosted_riddle_suite case requires profile."
|
|
17700
|
+
});
|
|
17701
|
+
continue;
|
|
17702
|
+
}
|
|
17703
|
+
let result;
|
|
17704
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
17705
|
+
const caseOptions = {
|
|
17706
|
+
...options,
|
|
17707
|
+
runner,
|
|
17708
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
17709
|
+
};
|
|
17710
|
+
try {
|
|
17711
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
17712
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
17713
|
+
options
|
|
17714
|
+
);
|
|
17715
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
17716
|
+
writeProfileOutput(caseOutputDir, result);
|
|
17717
|
+
} catch (error) {
|
|
17718
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17719
|
+
results.push({
|
|
17720
|
+
id: caseId,
|
|
17721
|
+
intent: cliString(testCase.intent) || null,
|
|
17722
|
+
ok: false,
|
|
17723
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17724
|
+
status: "configuration_error",
|
|
17725
|
+
error: message,
|
|
17726
|
+
output_dir: caseOutputDir || null
|
|
17727
|
+
});
|
|
17728
|
+
continue;
|
|
17729
|
+
}
|
|
17730
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
17731
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17732
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
17733
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
17734
|
+
const ok = statusOk && messageOk;
|
|
17735
|
+
results.push({
|
|
17736
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
17737
|
+
output_dir: caseOutputDir || null
|
|
17738
|
+
});
|
|
17739
|
+
}
|
|
17740
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17741
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17742
|
+
return {
|
|
17743
|
+
requested: true,
|
|
17744
|
+
configured: cases.length > 0,
|
|
17745
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
17746
|
+
runner,
|
|
17747
|
+
target,
|
|
17748
|
+
started_at: startedAt,
|
|
17749
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17750
|
+
case_count: cases.length,
|
|
17751
|
+
passed_case_count: results.length - failedCases.length,
|
|
17752
|
+
failed_cases: failedCases,
|
|
17753
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
17754
|
+
cases: results
|
|
17755
|
+
};
|
|
17756
|
+
}
|
|
17757
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
17758
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17759
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17760
|
+
const target = regressionPackRecord(suite.target);
|
|
17761
|
+
const lines = [
|
|
17762
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
17763
|
+
"",
|
|
17764
|
+
"Command:",
|
|
17765
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
17766
|
+
"",
|
|
17767
|
+
"Target:",
|
|
17768
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
17769
|
+
"",
|
|
17770
|
+
"Cases:",
|
|
17771
|
+
...cases.map((testCase, index) => {
|
|
17772
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17773
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
17774
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
17775
|
+
}),
|
|
17776
|
+
"",
|
|
17777
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
17778
|
+
].filter((line) => line !== "");
|
|
17779
|
+
return lines.join("\n");
|
|
17780
|
+
}
|
|
17542
17781
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
17543
17782
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
17544
17783
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -17560,6 +17799,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17560
17799
|
"- Run cases serially, not as one broad parallel batch.",
|
|
17561
17800
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
17562
17801
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
17802
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
17563
17803
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
17564
17804
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
17565
17805
|
"",
|
|
@@ -17572,12 +17812,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17572
17812
|
].join("\n");
|
|
17573
17813
|
}),
|
|
17574
17814
|
"",
|
|
17575
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
17815
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
17576
17816
|
].filter((line) => line !== "");
|
|
17577
17817
|
return lines.join("\n");
|
|
17578
17818
|
}
|
|
17579
17819
|
function compactRegressionPackRunResult(result) {
|
|
17580
17820
|
const localCore = regressionPackRecord(result.local_core);
|
|
17821
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17581
17822
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
17582
17823
|
return {
|
|
17583
17824
|
version: result.version,
|
|
@@ -17592,12 +17833,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
17592
17833
|
failed_cases: localCore.failed_cases,
|
|
17593
17834
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
17594
17835
|
},
|
|
17836
|
+
hosted_riddle: {
|
|
17837
|
+
requested: hostedRiddle.requested,
|
|
17838
|
+
configured: hostedRiddle.configured,
|
|
17839
|
+
ok: hostedRiddle.ok,
|
|
17840
|
+
runner: hostedRiddle.runner,
|
|
17841
|
+
case_count: hostedRiddle.case_count,
|
|
17842
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
17843
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
17844
|
+
},
|
|
17595
17845
|
openclaw_live_case_count: openClaw.case_count,
|
|
17596
17846
|
output_dir: result.output_dir
|
|
17597
17847
|
};
|
|
17598
17848
|
}
|
|
17599
17849
|
function regressionPackRunMarkdown(result) {
|
|
17600
17850
|
const localCore = regressionPackRecord(result.local_core);
|
|
17851
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17601
17852
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
17602
17853
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
17603
17854
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -17617,6 +17868,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
17617
17868
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
17618
17869
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
17619
17870
|
"",
|
|
17871
|
+
"## Hosted Riddle",
|
|
17872
|
+
"",
|
|
17873
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
17874
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
17875
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
17876
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
17877
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
17878
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
17879
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
17880
|
+
"",
|
|
17620
17881
|
"## Runtime Gate",
|
|
17621
17882
|
"",
|
|
17622
17883
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -17627,6 +17888,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
17627
17888
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
17628
17889
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
17629
17890
|
"",
|
|
17891
|
+
"## Hosted Riddle Handoff",
|
|
17892
|
+
"",
|
|
17893
|
+
"```text",
|
|
17894
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
17895
|
+
"```",
|
|
17896
|
+
"",
|
|
17630
17897
|
"## OC Handoff Prompt",
|
|
17631
17898
|
"",
|
|
17632
17899
|
"```text",
|
|
@@ -17643,22 +17910,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
17643
17910
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
17644
17911
|
`);
|
|
17645
17912
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
17913
|
+
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
17914
|
+
`);
|
|
17646
17915
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
17647
17916
|
`);
|
|
17648
17917
|
}
|
|
17649
|
-
function runRegressionPackForCli(options) {
|
|
17918
|
+
async function runRegressionPackForCli(options) {
|
|
17650
17919
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
17651
17920
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
17921
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
17652
17922
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
17923
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
17653
17924
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
17654
17925
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
17655
17926
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
17927
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
17656
17928
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
17657
|
-
const
|
|
17929
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
17930
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
17658
17931
|
const result = {
|
|
17659
17932
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
17660
17933
|
ok,
|
|
17661
17934
|
local_core_validated: localCoreValidated,
|
|
17935
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
17662
17936
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17663
17937
|
pack_path: filePath,
|
|
17664
17938
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -17668,13 +17942,19 @@ function runRegressionPackForCli(options) {
|
|
|
17668
17942
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
17669
17943
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
17670
17944
|
local_core: localCore,
|
|
17945
|
+
hosted_riddle: hostedRiddle,
|
|
17671
17946
|
openclaw_live_suite: {
|
|
17672
17947
|
target: regressionPackRecord(liveSuite.target),
|
|
17673
17948
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
17674
17949
|
case_count: liveCases.length,
|
|
17675
17950
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
17676
17951
|
},
|
|
17677
|
-
|
|
17952
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
17953
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
17954
|
+
localCoreOk: localCoreValidated,
|
|
17955
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
17956
|
+
hostedRiddleRequested
|
|
17957
|
+
}),
|
|
17678
17958
|
output_dir: profileOutputDirOption(options) || null
|
|
17679
17959
|
};
|
|
17680
17960
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -21361,7 +21641,7 @@ async function main() {
|
|
|
21361
21641
|
if (command === "regression-pack") {
|
|
21362
21642
|
const action = positional[1] || "run";
|
|
21363
21643
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
21364
|
-
const result = runRegressionPackForCli(options);
|
|
21644
|
+
const result = await runRegressionPackForCli(options);
|
|
21365
21645
|
writeRegressionPackRunResult(result, options);
|
|
21366
21646
|
process.exitCode = result.ok ? 0 : 1;
|
|
21367
21647
|
return;
|
package/dist/cli.js
CHANGED
|
@@ -40,6 +40,204 @@
|
|
|
40
40
|
"no-diff-prod-audit-default-capture-pass"
|
|
41
41
|
]
|
|
42
42
|
},
|
|
43
|
+
"hosted_riddle_suite": {
|
|
44
|
+
"runner": "riddle",
|
|
45
|
+
"target": {
|
|
46
|
+
"url": "https://riddledc.com/"
|
|
47
|
+
},
|
|
48
|
+
"result_log_fields": [
|
|
49
|
+
"case_id",
|
|
50
|
+
"profile_name",
|
|
51
|
+
"profile_status",
|
|
52
|
+
"route",
|
|
53
|
+
"riddle.job_id",
|
|
54
|
+
"riddle.status",
|
|
55
|
+
"artifacts",
|
|
56
|
+
"environment_blocker"
|
|
57
|
+
],
|
|
58
|
+
"cases": [
|
|
59
|
+
{
|
|
60
|
+
"id": "hosted-home-to-proof-route-change-pass",
|
|
61
|
+
"intent": "Run the generic hosted Riddle profile runner against production home, click Proof, and prove terminal /proof/ before involving OC.",
|
|
62
|
+
"profile": {
|
|
63
|
+
"version": "riddle-proof.profile.v1",
|
|
64
|
+
"name": "riddle-site-hosted-home-to-proof",
|
|
65
|
+
"target": {
|
|
66
|
+
"route": "/",
|
|
67
|
+
"wait_for_selector": "main#main-content",
|
|
68
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
69
|
+
"setup_actions": [
|
|
70
|
+
{ "type": "click", "selector": "a[href*='/proof']", "after_ms": 750 }
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
"checks": [
|
|
74
|
+
{ "type": "route_loaded", "expected_path": "/proof/" },
|
|
75
|
+
{ "type": "selector_visible", "selector": "main#main-content" },
|
|
76
|
+
{ "type": "text_visible", "text": "Proof" }
|
|
77
|
+
],
|
|
78
|
+
"metadata": {
|
|
79
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
80
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"expect": { "profile_status": "passed" }
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "hosted-proof-to-home-route-change-pass",
|
|
87
|
+
"intent": "Run the generic hosted Riddle profile runner from /proof/, click the root/Home link, and prove terminal /.",
|
|
88
|
+
"profile": {
|
|
89
|
+
"version": "riddle-proof.profile.v1",
|
|
90
|
+
"name": "riddle-site-hosted-proof-to-home",
|
|
91
|
+
"target": {
|
|
92
|
+
"route": "/proof/",
|
|
93
|
+
"wait_for_selector": "main#main-content",
|
|
94
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
95
|
+
"setup_actions": [
|
|
96
|
+
{ "type": "click", "selector": "a[href='/'], a[href='https://riddledc.com/']", "after_ms": 750 }
|
|
97
|
+
]
|
|
98
|
+
},
|
|
99
|
+
"checks": [
|
|
100
|
+
{ "type": "route_loaded", "expected_path": "/" },
|
|
101
|
+
{ "type": "selector_visible", "selector": "main#main-content" }
|
|
102
|
+
],
|
|
103
|
+
"metadata": {
|
|
104
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
105
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
"expect": { "profile_status": "passed" }
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "hosted-pricing-query-hash-positive-pass",
|
|
112
|
+
"intent": "Run the hosted profile runner directly against Pricing with query/hash and prove slash, query, and hash are preserved.",
|
|
113
|
+
"profile": {
|
|
114
|
+
"version": "riddle-proof.profile.v1",
|
|
115
|
+
"name": "riddle-site-hosted-pricing-query-hash",
|
|
116
|
+
"target": {
|
|
117
|
+
"route": "/pricing/?rp_probe=1#pricing-probe",
|
|
118
|
+
"wait_for_selector": "main#main-content",
|
|
119
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
120
|
+
"setup_actions": [
|
|
121
|
+
{
|
|
122
|
+
"type": "window_eval",
|
|
123
|
+
"script": "return window.location.hash;",
|
|
124
|
+
"expect_return": "#pricing-probe",
|
|
125
|
+
"store_return_to": "__riddleProof.pricingHash",
|
|
126
|
+
"capture_return": true
|
|
127
|
+
}
|
|
128
|
+
]
|
|
129
|
+
},
|
|
130
|
+
"checks": [
|
|
131
|
+
{ "type": "route_loaded", "expected_path": "/pricing/" },
|
|
132
|
+
{ "type": "url_search_param_equals", "param": "rp_probe", "expected_value": "1" },
|
|
133
|
+
{ "type": "selector_visible", "selector": "main#main-content" }
|
|
134
|
+
],
|
|
135
|
+
"metadata": {
|
|
136
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
137
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
"expect": { "profile_status": "passed" }
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"id": "hosted-pricing-query-hash-dropped-blocker",
|
|
144
|
+
"intent": "Run a hosted negative control where Pricing lacks the expected query/hash and require a product_regression profile result.",
|
|
145
|
+
"profile": {
|
|
146
|
+
"version": "riddle-proof.profile.v1",
|
|
147
|
+
"name": "riddle-site-hosted-pricing-query-hash-dropped",
|
|
148
|
+
"target": {
|
|
149
|
+
"route": "/pricing/",
|
|
150
|
+
"wait_for_selector": "main#main-content",
|
|
151
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
152
|
+
},
|
|
153
|
+
"checks": [
|
|
154
|
+
{ "type": "route_loaded", "expected_path": "/pricing/" },
|
|
155
|
+
{ "type": "url_search_param_equals", "param": "rp_probe", "expected_value": "1" }
|
|
156
|
+
],
|
|
157
|
+
"metadata": {
|
|
158
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
159
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
"expect": { "profile_status": "product_regression" }
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "hosted-no-diff-prod-audit-pass",
|
|
166
|
+
"intent": "Run a hosted no-diff production audit profile with no implementation or OC wrapper involved.",
|
|
167
|
+
"profile": {
|
|
168
|
+
"version": "riddle-proof.profile.v1",
|
|
169
|
+
"name": "riddle-site-hosted-no-diff-prod-audit",
|
|
170
|
+
"target": {
|
|
171
|
+
"route": "/",
|
|
172
|
+
"wait_for_selector": "main#main-content",
|
|
173
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
174
|
+
},
|
|
175
|
+
"checks": [
|
|
176
|
+
{ "type": "route_loaded", "expected_path": "/" },
|
|
177
|
+
{ "type": "selector_visible", "selector": "main#main-content" },
|
|
178
|
+
{ "type": "no_fatal_console_errors" }
|
|
179
|
+
],
|
|
180
|
+
"metadata": {
|
|
181
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
182
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
"expect": { "profile_status": "passed" }
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
"id": "hosted-missing-selector-timeout-blocker",
|
|
189
|
+
"intent": "Run a hosted profile with a missing selector and require a clean product_regression result, not a lifecycle failure.",
|
|
190
|
+
"profile": {
|
|
191
|
+
"version": "riddle-proof.profile.v1",
|
|
192
|
+
"name": "riddle-site-hosted-missing-selector",
|
|
193
|
+
"target": {
|
|
194
|
+
"route": "/",
|
|
195
|
+
"wait_for_selector": "main#main-content",
|
|
196
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
197
|
+
},
|
|
198
|
+
"checks": [
|
|
199
|
+
{ "type": "selector_visible", "selector": "#riddle-proof-missing-selector-timeout-smoke-20260604" }
|
|
200
|
+
],
|
|
201
|
+
"metadata": {
|
|
202
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
203
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
"expect": { "profile_status": "product_regression" }
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
"id": "hosted-thrown-error-specific-blocker",
|
|
210
|
+
"intent": "Run a hosted profile whose setup action throws a marker and require the marker to surface as a product_regression result.",
|
|
211
|
+
"profile": {
|
|
212
|
+
"version": "riddle-proof.profile.v1",
|
|
213
|
+
"name": "riddle-site-hosted-thrown-error",
|
|
214
|
+
"target": {
|
|
215
|
+
"route": "/",
|
|
216
|
+
"wait_for_selector": "main#main-content",
|
|
217
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
218
|
+
"setup_actions": [
|
|
219
|
+
{
|
|
220
|
+
"type": "window_eval",
|
|
221
|
+
"script": "throw new Error('RIDDLE_PROOF_HOSTED_THROWN_ERROR_SMOKE_20260604');",
|
|
222
|
+
"timeout_ms": 1000
|
|
223
|
+
}
|
|
224
|
+
]
|
|
225
|
+
},
|
|
226
|
+
"checks": [
|
|
227
|
+
{ "type": "route_loaded", "expected_path": "/" }
|
|
228
|
+
],
|
|
229
|
+
"metadata": {
|
|
230
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
231
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
232
|
+
}
|
|
233
|
+
},
|
|
234
|
+
"expect": {
|
|
235
|
+
"profile_status": "product_regression",
|
|
236
|
+
"message_contains": "RIDDLE_PROOF_HOSTED_THROWN_ERROR_SMOKE_20260604"
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
]
|
|
240
|
+
},
|
|
43
241
|
"openclaw_live_suite": {
|
|
44
242
|
"target": {
|
|
45
243
|
"repo": "davisdiehl/riddle-site",
|
package/package.json
CHANGED
package/runtime/lib/ship.py
CHANGED
|
@@ -320,7 +320,20 @@ def local_artifact_sources_fingerprint(local_sources):
|
|
|
320
320
|
def public_proof_artifacts(state):
|
|
321
321
|
published = state.get('proof_artifact_publication') or {}
|
|
322
322
|
artifacts = published.get('artifacts') if isinstance(published, dict) else []
|
|
323
|
-
|
|
323
|
+
attachments = state.get('proof_image_attachments') or {}
|
|
324
|
+
attachment_artifacts = attachments.get('artifacts') if isinstance(attachments, dict) else []
|
|
325
|
+
public = [
|
|
326
|
+
{
|
|
327
|
+
**artifact,
|
|
328
|
+
'raw_url': artifact.get('attachment_url'),
|
|
329
|
+
'html_url': artifact.get('attachment_url'),
|
|
330
|
+
'embeddable': True,
|
|
331
|
+
'attachment': True,
|
|
332
|
+
}
|
|
333
|
+
for artifact in attachment_artifacts or []
|
|
334
|
+
if isinstance(artifact, dict) and artifact.get('attachment_url')
|
|
335
|
+
]
|
|
336
|
+
public.extend([artifact for artifact in artifacts or [] if isinstance(artifact, dict) and (artifact.get('raw_url') or artifact.get('html_url'))])
|
|
324
337
|
for artifact in collect_proof_artifact_sources(state):
|
|
325
338
|
url = str(artifact.get('url') or '').strip()
|
|
326
339
|
if is_http_url(url):
|
|
@@ -328,6 +341,7 @@ def public_proof_artifacts(state):
|
|
|
328
341
|
**artifact,
|
|
329
342
|
'raw_url': url,
|
|
330
343
|
'html_url': url,
|
|
344
|
+
'embeddable': True,
|
|
331
345
|
'published': False,
|
|
332
346
|
})
|
|
333
347
|
deduped = []
|
|
@@ -346,6 +360,8 @@ def first_public_artifact_url(state, role, kind=None):
|
|
|
346
360
|
continue
|
|
347
361
|
if kind and artifact.get('kind') != kind:
|
|
348
362
|
continue
|
|
363
|
+
if artifact.get('kind') == 'image' and artifact.get('embeddable') is False:
|
|
364
|
+
continue
|
|
349
365
|
return artifact.get('raw_url') or artifact.get('html_url') or artifact.get('url') or ''
|
|
350
366
|
return ''
|
|
351
367
|
|
|
@@ -362,11 +378,92 @@ def resolve_github_repo_name(repo_dir):
|
|
|
362
378
|
return ''
|
|
363
379
|
|
|
364
380
|
|
|
381
|
+
def resolve_github_repo_private(repo_dir):
|
|
382
|
+
result = sp.run(['gh', 'repo', 'view', '--json', 'isPrivate', '--jq', '.isPrivate'],
|
|
383
|
+
cwd=repo_dir, capture_output=True, text=True, timeout=30)
|
|
384
|
+
if result.returncode == 0:
|
|
385
|
+
text = result.stdout.strip().lower()
|
|
386
|
+
if text in ('true', 'false'):
|
|
387
|
+
return text == 'true'
|
|
388
|
+
return None
|
|
389
|
+
|
|
390
|
+
|
|
365
391
|
def github_file_url(repo_name, ref, path_value, mode='blob'):
|
|
366
392
|
safe_path = urllib.parse.quote(str(path_value or '').lstrip('/'), safe='/._-')
|
|
367
393
|
return 'https://github.com/' + repo_name + '/' + mode + '/' + ref + '/' + safe_path
|
|
368
394
|
|
|
369
395
|
|
|
396
|
+
def parse_github_image_markdown(line):
|
|
397
|
+
match = re.search(r'!\[[^\]]*\]\((https://github\.com/user-attachments/assets/[^)\s]+)\)', str(line or '').strip())
|
|
398
|
+
return match.group(1) if match else ''
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def upload_local_images_to_github_attachments(state, repo_dir):
|
|
402
|
+
if truthy(os.environ.get('RIDDLE_PROOF_DISABLE_GITHUB_IMAGE_ATTACHMENTS')):
|
|
403
|
+
state['proof_image_attachments'] = {'ok': False, 'skipped': True, 'reason': 'disabled'}
|
|
404
|
+
save_state(state)
|
|
405
|
+
return state['proof_image_attachments']
|
|
406
|
+
|
|
407
|
+
image_sources = [
|
|
408
|
+
artifact for artifact in local_proof_artifact_sources(state)
|
|
409
|
+
if artifact.get('kind') == 'image' and artifact.get('path')
|
|
410
|
+
]
|
|
411
|
+
if not image_sources:
|
|
412
|
+
state['proof_image_attachments'] = {'ok': True, 'skipped': True, 'reason': 'no local image artifacts'}
|
|
413
|
+
save_state(state)
|
|
414
|
+
return state['proof_image_attachments']
|
|
415
|
+
|
|
416
|
+
repo_name = resolve_github_repo_name(repo_dir)
|
|
417
|
+
if not repo_name:
|
|
418
|
+
state['proof_image_attachments'] = {'ok': False, 'skipped': True, 'reason': 'repo name unavailable'}
|
|
419
|
+
save_state(state)
|
|
420
|
+
return state['proof_image_attachments']
|
|
421
|
+
|
|
422
|
+
timeout_seconds = int(os.environ.get('RIDDLE_PROOF_GH_IMAGE_TIMEOUT_SECONDS') or '15')
|
|
423
|
+
command = ['gh', 'image'] + [artifact.get('path') for artifact in image_sources] + ['--repo', repo_name]
|
|
424
|
+
try:
|
|
425
|
+
result = sp.run(command, cwd=repo_dir, capture_output=True, text=True, timeout=timeout_seconds)
|
|
426
|
+
except sp.TimeoutExpired:
|
|
427
|
+
state['proof_image_attachments'] = {'ok': False, 'skipped': True, 'reason': 'gh image timed out'}
|
|
428
|
+
save_state(state)
|
|
429
|
+
return state['proof_image_attachments']
|
|
430
|
+
|
|
431
|
+
if result.returncode != 0:
|
|
432
|
+
state['proof_image_attachments'] = {
|
|
433
|
+
'ok': False,
|
|
434
|
+
'skipped': True,
|
|
435
|
+
'reason': 'gh image failed',
|
|
436
|
+
'stderr': (result.stderr or result.stdout or '')[:500],
|
|
437
|
+
}
|
|
438
|
+
save_state(state)
|
|
439
|
+
return state['proof_image_attachments']
|
|
440
|
+
|
|
441
|
+
urls = [parse_github_image_markdown(line) for line in result.stdout.splitlines()]
|
|
442
|
+
urls = [url for url in urls if url]
|
|
443
|
+
if not urls:
|
|
444
|
+
state['proof_image_attachments'] = {'ok': False, 'skipped': True, 'reason': 'gh image returned no attachment URLs'}
|
|
445
|
+
save_state(state)
|
|
446
|
+
return state['proof_image_attachments']
|
|
447
|
+
|
|
448
|
+
artifacts = []
|
|
449
|
+
for index, artifact in enumerate(image_sources):
|
|
450
|
+
if index >= len(urls):
|
|
451
|
+
break
|
|
452
|
+
artifacts.append({
|
|
453
|
+
**artifact,
|
|
454
|
+
'attachment_url': urls[index],
|
|
455
|
+
'attachment': True,
|
|
456
|
+
'embeddable': True,
|
|
457
|
+
})
|
|
458
|
+
state['proof_image_attachments'] = {
|
|
459
|
+
'ok': True,
|
|
460
|
+
'repo': repo_name,
|
|
461
|
+
'artifacts': artifacts,
|
|
462
|
+
}
|
|
463
|
+
save_state(state)
|
|
464
|
+
return state['proof_image_attachments']
|
|
465
|
+
|
|
466
|
+
|
|
370
467
|
def write_artifact_readme(path_value, state, artifacts):
|
|
371
468
|
lines = [
|
|
372
469
|
'# Riddle Proof Artifacts',
|
|
@@ -408,6 +505,7 @@ def publish_local_proof_artifacts_to_github(state, repo_dir, pr_num):
|
|
|
408
505
|
repo_name = resolve_github_repo_name(repo_dir)
|
|
409
506
|
if not repo_name:
|
|
410
507
|
raise SystemExit('Could not resolve GitHub repository name for proof artifact publication.')
|
|
508
|
+
repo_private = resolve_github_repo_private(repo_dir)
|
|
411
509
|
|
|
412
510
|
run_id = safe_slug(state.get('run_id') or str(int(time.time())), 'run')
|
|
413
511
|
pr_slug = safe_slug('pr-' + str(pr_num or 'unknown'), 'pr')
|
|
@@ -474,11 +572,13 @@ def publish_local_proof_artifacts_to_github(state, repo_dir, pr_num):
|
|
|
474
572
|
published_path = artifact.get('published_path')
|
|
475
573
|
artifact['raw_url'] = github_file_url(repo_name, commit, published_path, 'raw')
|
|
476
574
|
artifact['html_url'] = github_file_url(repo_name, commit, published_path, 'blob')
|
|
575
|
+
artifact['embeddable'] = False if repo_private is True and artifact.get('kind') == 'image' else True
|
|
477
576
|
publication = {
|
|
478
577
|
'ok': True,
|
|
479
578
|
'branch': artifact_branch,
|
|
480
579
|
'commit': commit,
|
|
481
580
|
'repo': repo_name,
|
|
581
|
+
'repo_private': repo_private,
|
|
482
582
|
'html_url': github_file_url(repo_name, commit, artifact_dir_name, 'tree'),
|
|
483
583
|
'manifest_url': github_file_url(repo_name, commit, artifact_dir_name + '/proof-artifacts.json', 'blob'),
|
|
484
584
|
'readme_url': github_file_url(repo_name, commit, artifact_dir_name + '/README.md', 'blob'),
|
|
@@ -1080,6 +1180,12 @@ if publication.get('ok') and not publication.get('skipped'):
|
|
|
1080
1180
|
elif publication.get('skipped'):
|
|
1081
1181
|
print('Proof artifact publication skipped: ' + publication.get('reason', 'unknown'))
|
|
1082
1182
|
|
|
1183
|
+
image_attachments = upload_local_images_to_github_attachments(s, repo_dir)
|
|
1184
|
+
if image_attachments.get('ok') and not image_attachments.get('skipped'):
|
|
1185
|
+
print('Proof images attached: ' + str(len(image_attachments.get('artifacts') or [])))
|
|
1186
|
+
elif image_attachments.get('skipped'):
|
|
1187
|
+
print('Proof image attachment skipped: ' + image_attachments.get('reason', 'unknown'))
|
|
1188
|
+
|
|
1083
1189
|
# Post proof comment on PR
|
|
1084
1190
|
body = '## Riddle Proof — Proof of Fix\n\n'
|
|
1085
1191
|
body += '**Goal:** ' + s.get('change_request', '') + '\n\n'
|
|
@@ -1099,15 +1205,34 @@ if publication.get('ok') and not publication.get('skipped'):
|
|
|
1099
1205
|
before_image = first_public_artifact_url(s, 'before', 'image')
|
|
1100
1206
|
prod_image = first_public_artifact_url(s, 'prod', 'image')
|
|
1101
1207
|
after_image = first_public_artifact_url(s, 'after', 'image')
|
|
1208
|
+
linked_image_artifacts = [
|
|
1209
|
+
artifact for artifact in public_artifacts
|
|
1210
|
+
if artifact.get('kind') == 'image'
|
|
1211
|
+
and artifact.get('embeddable') is False
|
|
1212
|
+
and (artifact.get('html_url') or artifact.get('raw_url'))
|
|
1213
|
+
]
|
|
1102
1214
|
if before_image:
|
|
1103
1215
|
body += '### Before\n\n\n'
|
|
1104
1216
|
if prod_image:
|
|
1105
1217
|
body += '### Prod\n\n\n'
|
|
1106
1218
|
if after_image:
|
|
1107
1219
|
body += '### After\n\n\n'
|
|
1220
|
+
elif linked_image_artifacts:
|
|
1221
|
+
body += '### After evidence\nA screenshot artifact was captured, but GitHub attachment upload was unavailable; linked image artifacts are listed below.\n\n'
|
|
1108
1222
|
else:
|
|
1109
1223
|
body += '### After evidence\nNo after screenshot was captured for this verification mode; structured evidence is summarized below.\n\n'
|
|
1110
1224
|
|
|
1225
|
+
if linked_image_artifacts:
|
|
1226
|
+
body += '### Image artifacts\n'
|
|
1227
|
+
for artifact in linked_image_artifacts[:12]:
|
|
1228
|
+
label = artifact.get('name') or artifact.get('filename') or 'image'
|
|
1229
|
+
url = artifact.get('html_url') or artifact.get('raw_url')
|
|
1230
|
+
body += '- [' + str(label) + '](' + str(url) + ')'
|
|
1231
|
+
if artifact.get('role'):
|
|
1232
|
+
body += ' (' + str(artifact.get('role')) + ')'
|
|
1233
|
+
body += '\n'
|
|
1234
|
+
body += '\n'
|
|
1235
|
+
|
|
1111
1236
|
data_artifacts = [
|
|
1112
1237
|
artifact for artifact in public_artifacts
|
|
1113
1238
|
if artifact.get('kind') != 'image' and (artifact.get('html_url') or artifact.get('raw_url'))
|
|
@@ -27,7 +27,15 @@ import sys
|
|
|
27
27
|
|
|
28
28
|
args = sys.argv[1:]
|
|
29
29
|
if args[:3] == ["repo", "view", "--json"]:
|
|
30
|
-
|
|
30
|
+
if "isPrivate" in args:
|
|
31
|
+
print("true")
|
|
32
|
+
else:
|
|
33
|
+
print("example/test-repo")
|
|
34
|
+
raise SystemExit(0)
|
|
35
|
+
if args and args[0] == "image":
|
|
36
|
+
image_paths = [arg for arg in args[1:] if not arg.startswith("--") and arg != "example/test-repo"]
|
|
37
|
+
for index, image_path in enumerate(image_paths):
|
|
38
|
+
print(f"")
|
|
31
39
|
raise SystemExit(0)
|
|
32
40
|
if args[:2] == ["pr", "list"]:
|
|
33
41
|
print("")
|
|
@@ -163,16 +171,16 @@ def main():
|
|
|
163
171
|
assert publication.get("ok") is True, "proof artifact publication should be recorded"
|
|
164
172
|
assert publication.get("artifacts"), "published artifact list should be recorded"
|
|
165
173
|
assert updated.get("ship_report", {}).get("after_artifact_url", "").startswith(
|
|
166
|
-
"https://github.com/
|
|
167
|
-
), "ship report should expose a GitHub-hosted after artifact
|
|
174
|
+
"https://github.com/user-attachments/assets/"
|
|
175
|
+
), "ship report should expose a GitHub-hosted attachment URL for the after artifact"
|
|
168
176
|
|
|
169
177
|
comment = comment_body_path.read_text(encoding="utf-8")
|
|
170
178
|
assert "file://" not in comment, "PR proof comment must not expose local file URLs"
|
|
171
179
|
assert "raw.githubusercontent.com" not in comment, (
|
|
172
180
|
"PR proof comment must not depend on unauthenticated raw GitHub URLs"
|
|
173
181
|
)
|
|
174
|
-
assert "
|
|
177
185
|
assert "[proof.json](https://github.com/example/test-repo/blob/" in comment, (
|
|
178
186
|
"PR proof comment should link the structured proof JSON"
|