@riddledc/riddle-proof 0.8.34 → 0.8.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -65,6 +65,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
65
65
|
"format",
|
|
66
66
|
"framework",
|
|
67
67
|
"help",
|
|
68
|
+
"hostedRiddle",
|
|
68
69
|
"image",
|
|
69
70
|
"input",
|
|
70
71
|
"inputDir",
|
|
@@ -136,7 +137,7 @@ function usage() {
|
|
|
136
137
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
137
138
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
138
139
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
139
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
141
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
141
142
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
142
143
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -447,6 +448,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
447
448
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
448
449
|
};
|
|
449
450
|
}
|
|
451
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
452
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
453
|
+
}
|
|
454
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
455
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
456
|
+
}
|
|
457
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
458
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
459
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
460
|
+
}
|
|
461
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
462
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
463
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
464
|
+
}
|
|
465
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
466
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
467
|
+
return cliString(expect.message_contains);
|
|
468
|
+
}
|
|
469
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
470
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
471
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
472
|
+
return {
|
|
473
|
+
requested: false,
|
|
474
|
+
configured: cases.length > 0,
|
|
475
|
+
ok: true,
|
|
476
|
+
runner: cliString(suite.runner) || "riddle",
|
|
477
|
+
target: regressionPackRecord(suite.target),
|
|
478
|
+
case_count: cases.length,
|
|
479
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
483
|
+
return outputDir ? path.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
484
|
+
}
|
|
485
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
486
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
487
|
+
return {
|
|
488
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
489
|
+
intent: cliString(testCase.intent) || null,
|
|
490
|
+
ok,
|
|
491
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
492
|
+
expected_message_contains: expectedMessage || null,
|
|
493
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
494
|
+
status: result.status,
|
|
495
|
+
profile_name: result.profile_name,
|
|
496
|
+
summary: result.summary,
|
|
497
|
+
route: result.route,
|
|
498
|
+
riddle: result.riddle,
|
|
499
|
+
artifacts: result.artifacts,
|
|
500
|
+
environment_blocker: result.environment_blocker,
|
|
501
|
+
error: result.error
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
505
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
506
|
+
return {
|
|
507
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
508
|
+
intent: cliString(testCase.intent) || null,
|
|
509
|
+
ok: false,
|
|
510
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
511
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
512
|
+
status: "environment_blocked",
|
|
513
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
514
|
+
summary: error,
|
|
515
|
+
environment_blocker: environmentBlocker,
|
|
516
|
+
error
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
520
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
521
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
522
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
523
|
+
let balance;
|
|
524
|
+
try {
|
|
525
|
+
balance = await client.getBalance();
|
|
526
|
+
} catch (error) {
|
|
527
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
528
|
+
return {
|
|
529
|
+
error: message,
|
|
530
|
+
environment_blocker: {
|
|
531
|
+
source: "riddle_api",
|
|
532
|
+
endpoint: "/v1/balance",
|
|
533
|
+
reason: "balance_preflight_failed",
|
|
534
|
+
balance_preflight: true,
|
|
535
|
+
job_count: cases.length,
|
|
536
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
537
|
+
required_seconds: requiredSeconds,
|
|
538
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
539
|
+
...apiKeySourceBlockerMetadata(client)
|
|
540
|
+
}
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
544
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
545
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
546
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
547
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
548
|
+
return {
|
|
549
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
550
|
+
environment_blocker: {
|
|
551
|
+
source: "riddle_api",
|
|
552
|
+
endpoint: "/v1/balance",
|
|
553
|
+
reason: "insufficient_balance",
|
|
554
|
+
error: "Insufficient available balance",
|
|
555
|
+
balance_preflight: true,
|
|
556
|
+
job_count: cases.length,
|
|
557
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
558
|
+
required_seconds: requiredSeconds,
|
|
559
|
+
available_seconds: availableSeconds,
|
|
560
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
561
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
562
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
563
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
564
|
+
...apiKeySourceBlockerMetadata(client)
|
|
565
|
+
}
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
569
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
570
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
571
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
572
|
+
const target = regressionPackRecord(suite.target);
|
|
573
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
574
|
+
const outputDir = profileOutputDirOption(options);
|
|
575
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
576
|
+
const results = [];
|
|
577
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
578
|
+
if (suiteBlocker) {
|
|
579
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
580
|
+
return {
|
|
581
|
+
requested: true,
|
|
582
|
+
configured: cases.length > 0,
|
|
583
|
+
ok: false,
|
|
584
|
+
runner,
|
|
585
|
+
target,
|
|
586
|
+
started_at: startedAt,
|
|
587
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
588
|
+
case_count: cases.length,
|
|
589
|
+
passed_case_count: 0,
|
|
590
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
591
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
592
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
593
|
+
error: suiteBlocker.error,
|
|
594
|
+
cases: blockedCases
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
for (const testCase of cases) {
|
|
598
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
599
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
600
|
+
if (!Object.keys(rawProfile).length) {
|
|
601
|
+
results.push({
|
|
602
|
+
id: caseId,
|
|
603
|
+
intent: cliString(testCase.intent) || null,
|
|
604
|
+
ok: false,
|
|
605
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
606
|
+
status: "configuration_error",
|
|
607
|
+
error: "hosted_riddle_suite case requires profile."
|
|
608
|
+
});
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
let result;
|
|
612
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
613
|
+
const caseOptions = {
|
|
614
|
+
...options,
|
|
615
|
+
runner,
|
|
616
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
617
|
+
};
|
|
618
|
+
try {
|
|
619
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
620
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
621
|
+
options
|
|
622
|
+
);
|
|
623
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
624
|
+
writeProfileOutput(caseOutputDir, result);
|
|
625
|
+
} catch (error) {
|
|
626
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
627
|
+
results.push({
|
|
628
|
+
id: caseId,
|
|
629
|
+
intent: cliString(testCase.intent) || null,
|
|
630
|
+
ok: false,
|
|
631
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
632
|
+
status: "configuration_error",
|
|
633
|
+
error: message,
|
|
634
|
+
output_dir: caseOutputDir || null
|
|
635
|
+
});
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
639
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
640
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
641
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
642
|
+
const ok = statusOk && messageOk;
|
|
643
|
+
results.push({
|
|
644
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
645
|
+
output_dir: caseOutputDir || null
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
649
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
650
|
+
return {
|
|
651
|
+
requested: true,
|
|
652
|
+
configured: cases.length > 0,
|
|
653
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
654
|
+
runner,
|
|
655
|
+
target,
|
|
656
|
+
started_at: startedAt,
|
|
657
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
658
|
+
case_count: cases.length,
|
|
659
|
+
passed_case_count: results.length - failedCases.length,
|
|
660
|
+
failed_cases: failedCases,
|
|
661
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
662
|
+
cases: results
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
666
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
667
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
668
|
+
const target = regressionPackRecord(suite.target);
|
|
669
|
+
const lines = [
|
|
670
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
671
|
+
"",
|
|
672
|
+
"Command:",
|
|
673
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
674
|
+
"",
|
|
675
|
+
"Target:",
|
|
676
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
677
|
+
"",
|
|
678
|
+
"Cases:",
|
|
679
|
+
...cases.map((testCase, index) => {
|
|
680
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
681
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
682
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
683
|
+
}),
|
|
684
|
+
"",
|
|
685
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
686
|
+
].filter((line) => line !== "");
|
|
687
|
+
return lines.join("\n");
|
|
688
|
+
}
|
|
450
689
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
451
690
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
452
691
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -468,6 +707,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
468
707
|
"- Run cases serially, not as one broad parallel batch.",
|
|
469
708
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
470
709
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
710
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
471
711
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
472
712
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
473
713
|
"",
|
|
@@ -480,12 +720,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
480
720
|
].join("\n");
|
|
481
721
|
}),
|
|
482
722
|
"",
|
|
483
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
723
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
484
724
|
].filter((line) => line !== "");
|
|
485
725
|
return lines.join("\n");
|
|
486
726
|
}
|
|
487
727
|
function compactRegressionPackRunResult(result) {
|
|
488
728
|
const localCore = regressionPackRecord(result.local_core);
|
|
729
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
489
730
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
490
731
|
return {
|
|
491
732
|
version: result.version,
|
|
@@ -500,12 +741,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
500
741
|
failed_cases: localCore.failed_cases,
|
|
501
742
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
502
743
|
},
|
|
744
|
+
hosted_riddle: {
|
|
745
|
+
requested: hostedRiddle.requested,
|
|
746
|
+
configured: hostedRiddle.configured,
|
|
747
|
+
ok: hostedRiddle.ok,
|
|
748
|
+
runner: hostedRiddle.runner,
|
|
749
|
+
case_count: hostedRiddle.case_count,
|
|
750
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
751
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
752
|
+
},
|
|
503
753
|
openclaw_live_case_count: openClaw.case_count,
|
|
504
754
|
output_dir: result.output_dir
|
|
505
755
|
};
|
|
506
756
|
}
|
|
507
757
|
function regressionPackRunMarkdown(result) {
|
|
508
758
|
const localCore = regressionPackRecord(result.local_core);
|
|
759
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
509
760
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
510
761
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
511
762
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -525,6 +776,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
525
776
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
526
777
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
527
778
|
"",
|
|
779
|
+
"## Hosted Riddle",
|
|
780
|
+
"",
|
|
781
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
782
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
783
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
784
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
785
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
786
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
787
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
788
|
+
"",
|
|
528
789
|
"## Runtime Gate",
|
|
529
790
|
"",
|
|
530
791
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -535,6 +796,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
535
796
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
536
797
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
537
798
|
"",
|
|
799
|
+
"## Hosted Riddle Handoff",
|
|
800
|
+
"",
|
|
801
|
+
"```text",
|
|
802
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
803
|
+
"```",
|
|
804
|
+
"",
|
|
538
805
|
"## OC Handoff Prompt",
|
|
539
806
|
"",
|
|
540
807
|
"```text",
|
|
@@ -551,22 +818,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
551
818
|
writeFileSync(path.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
552
819
|
`);
|
|
553
820
|
writeFileSync(path.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
821
|
+
writeFileSync(path.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
822
|
+
`);
|
|
554
823
|
writeFileSync(path.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
555
824
|
`);
|
|
556
825
|
}
|
|
557
|
-
function runRegressionPackForCli(options) {
|
|
826
|
+
async function runRegressionPackForCli(options) {
|
|
558
827
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
559
828
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
829
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
560
830
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
831
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
561
832
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
562
833
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
563
834
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
835
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
564
836
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
565
|
-
const
|
|
837
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
838
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
566
839
|
const result = {
|
|
567
840
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
568
841
|
ok,
|
|
569
842
|
local_core_validated: localCoreValidated,
|
|
843
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
570
844
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
571
845
|
pack_path: filePath,
|
|
572
846
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -576,13 +850,19 @@ function runRegressionPackForCli(options) {
|
|
|
576
850
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
577
851
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
578
852
|
local_core: localCore,
|
|
853
|
+
hosted_riddle: hostedRiddle,
|
|
579
854
|
openclaw_live_suite: {
|
|
580
855
|
target: regressionPackRecord(liveSuite.target),
|
|
581
856
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
582
857
|
case_count: liveCases.length,
|
|
583
858
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
584
859
|
},
|
|
585
|
-
|
|
860
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
861
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
862
|
+
localCoreOk: localCoreValidated,
|
|
863
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
864
|
+
hostedRiddleRequested
|
|
865
|
+
}),
|
|
586
866
|
output_dir: profileOutputDirOption(options) || null
|
|
587
867
|
};
|
|
588
868
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -4269,7 +4549,7 @@ async function main() {
|
|
|
4269
4549
|
if (command === "regression-pack") {
|
|
4270
4550
|
const action = positional[1] || "run";
|
|
4271
4551
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
4272
|
-
const result = runRegressionPackForCli(options);
|
|
4552
|
+
const result = await runRegressionPackForCli(options);
|
|
4273
4553
|
writeRegressionPackRunResult(result, options);
|
|
4274
4554
|
process.exitCode = result.ok ? 0 : 1;
|
|
4275
4555
|
return;
|
package/dist/cli/index.js
CHANGED
package/dist/cli.cjs
CHANGED
|
@@ -17157,6 +17157,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
17157
17157
|
"format",
|
|
17158
17158
|
"framework",
|
|
17159
17159
|
"help",
|
|
17160
|
+
"hostedRiddle",
|
|
17160
17161
|
"image",
|
|
17161
17162
|
"input",
|
|
17162
17163
|
"inputDir",
|
|
@@ -17228,7 +17229,7 @@ function usage() {
|
|
|
17228
17229
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
17229
17230
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17230
17231
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17231
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17232
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17232
17233
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
17233
17234
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
17234
17235
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -17539,6 +17540,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
17539
17540
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
17540
17541
|
};
|
|
17541
17542
|
}
|
|
17543
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
17544
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
17545
|
+
}
|
|
17546
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
17547
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
17548
|
+
}
|
|
17549
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
17550
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17551
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
17552
|
+
}
|
|
17553
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
17554
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17555
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
17556
|
+
}
|
|
17557
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
17558
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17559
|
+
return cliString(expect.message_contains);
|
|
17560
|
+
}
|
|
17561
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
17562
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17563
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17564
|
+
return {
|
|
17565
|
+
requested: false,
|
|
17566
|
+
configured: cases.length > 0,
|
|
17567
|
+
ok: true,
|
|
17568
|
+
runner: cliString(suite.runner) || "riddle",
|
|
17569
|
+
target: regressionPackRecord(suite.target),
|
|
17570
|
+
case_count: cases.length,
|
|
17571
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
17572
|
+
};
|
|
17573
|
+
}
|
|
17574
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
17575
|
+
return outputDir ? import_node_path6.default.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
17576
|
+
}
|
|
17577
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
17578
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17579
|
+
return {
|
|
17580
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
17581
|
+
intent: cliString(testCase.intent) || null,
|
|
17582
|
+
ok,
|
|
17583
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17584
|
+
expected_message_contains: expectedMessage || null,
|
|
17585
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
17586
|
+
status: result.status,
|
|
17587
|
+
profile_name: result.profile_name,
|
|
17588
|
+
summary: result.summary,
|
|
17589
|
+
route: result.route,
|
|
17590
|
+
riddle: result.riddle,
|
|
17591
|
+
artifacts: result.artifacts,
|
|
17592
|
+
environment_blocker: result.environment_blocker,
|
|
17593
|
+
error: result.error
|
|
17594
|
+
};
|
|
17595
|
+
}
|
|
17596
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
17597
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17598
|
+
return {
|
|
17599
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
17600
|
+
intent: cliString(testCase.intent) || null,
|
|
17601
|
+
ok: false,
|
|
17602
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17603
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
17604
|
+
status: "environment_blocked",
|
|
17605
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
17606
|
+
summary: error,
|
|
17607
|
+
environment_blocker: environmentBlocker,
|
|
17608
|
+
error
|
|
17609
|
+
};
|
|
17610
|
+
}
|
|
17611
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
17612
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
17613
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
17614
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
17615
|
+
let balance;
|
|
17616
|
+
try {
|
|
17617
|
+
balance = await client.getBalance();
|
|
17618
|
+
} catch (error) {
|
|
17619
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17620
|
+
return {
|
|
17621
|
+
error: message,
|
|
17622
|
+
environment_blocker: {
|
|
17623
|
+
source: "riddle_api",
|
|
17624
|
+
endpoint: "/v1/balance",
|
|
17625
|
+
reason: "balance_preflight_failed",
|
|
17626
|
+
balance_preflight: true,
|
|
17627
|
+
job_count: cases.length,
|
|
17628
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17629
|
+
required_seconds: requiredSeconds,
|
|
17630
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
17631
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17632
|
+
}
|
|
17633
|
+
};
|
|
17634
|
+
}
|
|
17635
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
17636
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
17637
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
17638
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
17639
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
17640
|
+
return {
|
|
17641
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
17642
|
+
environment_blocker: {
|
|
17643
|
+
source: "riddle_api",
|
|
17644
|
+
endpoint: "/v1/balance",
|
|
17645
|
+
reason: "insufficient_balance",
|
|
17646
|
+
error: "Insufficient available balance",
|
|
17647
|
+
balance_preflight: true,
|
|
17648
|
+
job_count: cases.length,
|
|
17649
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17650
|
+
required_seconds: requiredSeconds,
|
|
17651
|
+
available_seconds: availableSeconds,
|
|
17652
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
17653
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
17654
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
17655
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
17656
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17657
|
+
}
|
|
17658
|
+
};
|
|
17659
|
+
}
|
|
17660
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
17661
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17662
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17663
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
17664
|
+
const target = regressionPackRecord(suite.target);
|
|
17665
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
17666
|
+
const outputDir = profileOutputDirOption(options);
|
|
17667
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17668
|
+
const results = [];
|
|
17669
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
17670
|
+
if (suiteBlocker) {
|
|
17671
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
17672
|
+
return {
|
|
17673
|
+
requested: true,
|
|
17674
|
+
configured: cases.length > 0,
|
|
17675
|
+
ok: false,
|
|
17676
|
+
runner,
|
|
17677
|
+
target,
|
|
17678
|
+
started_at: startedAt,
|
|
17679
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17680
|
+
case_count: cases.length,
|
|
17681
|
+
passed_case_count: 0,
|
|
17682
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17683
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17684
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
17685
|
+
error: suiteBlocker.error,
|
|
17686
|
+
cases: blockedCases
|
|
17687
|
+
};
|
|
17688
|
+
}
|
|
17689
|
+
for (const testCase of cases) {
|
|
17690
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
17691
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17692
|
+
if (!Object.keys(rawProfile).length) {
|
|
17693
|
+
results.push({
|
|
17694
|
+
id: caseId,
|
|
17695
|
+
intent: cliString(testCase.intent) || null,
|
|
17696
|
+
ok: false,
|
|
17697
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17698
|
+
status: "configuration_error",
|
|
17699
|
+
error: "hosted_riddle_suite case requires profile."
|
|
17700
|
+
});
|
|
17701
|
+
continue;
|
|
17702
|
+
}
|
|
17703
|
+
let result;
|
|
17704
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
17705
|
+
const caseOptions = {
|
|
17706
|
+
...options,
|
|
17707
|
+
runner,
|
|
17708
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
17709
|
+
};
|
|
17710
|
+
try {
|
|
17711
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
17712
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
17713
|
+
options
|
|
17714
|
+
);
|
|
17715
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
17716
|
+
writeProfileOutput(caseOutputDir, result);
|
|
17717
|
+
} catch (error) {
|
|
17718
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17719
|
+
results.push({
|
|
17720
|
+
id: caseId,
|
|
17721
|
+
intent: cliString(testCase.intent) || null,
|
|
17722
|
+
ok: false,
|
|
17723
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17724
|
+
status: "configuration_error",
|
|
17725
|
+
error: message,
|
|
17726
|
+
output_dir: caseOutputDir || null
|
|
17727
|
+
});
|
|
17728
|
+
continue;
|
|
17729
|
+
}
|
|
17730
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
17731
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17732
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
17733
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
17734
|
+
const ok = statusOk && messageOk;
|
|
17735
|
+
results.push({
|
|
17736
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
17737
|
+
output_dir: caseOutputDir || null
|
|
17738
|
+
});
|
|
17739
|
+
}
|
|
17740
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17741
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17742
|
+
return {
|
|
17743
|
+
requested: true,
|
|
17744
|
+
configured: cases.length > 0,
|
|
17745
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
17746
|
+
runner,
|
|
17747
|
+
target,
|
|
17748
|
+
started_at: startedAt,
|
|
17749
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17750
|
+
case_count: cases.length,
|
|
17751
|
+
passed_case_count: results.length - failedCases.length,
|
|
17752
|
+
failed_cases: failedCases,
|
|
17753
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
17754
|
+
cases: results
|
|
17755
|
+
};
|
|
17756
|
+
}
|
|
17757
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
17758
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17759
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17760
|
+
const target = regressionPackRecord(suite.target);
|
|
17761
|
+
const lines = [
|
|
17762
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
17763
|
+
"",
|
|
17764
|
+
"Command:",
|
|
17765
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
17766
|
+
"",
|
|
17767
|
+
"Target:",
|
|
17768
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
17769
|
+
"",
|
|
17770
|
+
"Cases:",
|
|
17771
|
+
...cases.map((testCase, index) => {
|
|
17772
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17773
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
17774
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
17775
|
+
}),
|
|
17776
|
+
"",
|
|
17777
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
17778
|
+
].filter((line) => line !== "");
|
|
17779
|
+
return lines.join("\n");
|
|
17780
|
+
}
|
|
17542
17781
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
17543
17782
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
17544
17783
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -17560,6 +17799,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17560
17799
|
"- Run cases serially, not as one broad parallel batch.",
|
|
17561
17800
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
17562
17801
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
17802
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
17563
17803
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
17564
17804
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
17565
17805
|
"",
|
|
@@ -17572,12 +17812,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17572
17812
|
].join("\n");
|
|
17573
17813
|
}),
|
|
17574
17814
|
"",
|
|
17575
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
17815
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
17576
17816
|
].filter((line) => line !== "");
|
|
17577
17817
|
return lines.join("\n");
|
|
17578
17818
|
}
|
|
17579
17819
|
function compactRegressionPackRunResult(result) {
|
|
17580
17820
|
const localCore = regressionPackRecord(result.local_core);
|
|
17821
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17581
17822
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
17582
17823
|
return {
|
|
17583
17824
|
version: result.version,
|
|
@@ -17592,12 +17833,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
17592
17833
|
failed_cases: localCore.failed_cases,
|
|
17593
17834
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
17594
17835
|
},
|
|
17836
|
+
hosted_riddle: {
|
|
17837
|
+
requested: hostedRiddle.requested,
|
|
17838
|
+
configured: hostedRiddle.configured,
|
|
17839
|
+
ok: hostedRiddle.ok,
|
|
17840
|
+
runner: hostedRiddle.runner,
|
|
17841
|
+
case_count: hostedRiddle.case_count,
|
|
17842
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
17843
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
17844
|
+
},
|
|
17595
17845
|
openclaw_live_case_count: openClaw.case_count,
|
|
17596
17846
|
output_dir: result.output_dir
|
|
17597
17847
|
};
|
|
17598
17848
|
}
|
|
17599
17849
|
function regressionPackRunMarkdown(result) {
|
|
17600
17850
|
const localCore = regressionPackRecord(result.local_core);
|
|
17851
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17601
17852
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
17602
17853
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
17603
17854
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -17617,6 +17868,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
17617
17868
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
17618
17869
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
17619
17870
|
"",
|
|
17871
|
+
"## Hosted Riddle",
|
|
17872
|
+
"",
|
|
17873
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
17874
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
17875
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
17876
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
17877
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
17878
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
17879
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
17880
|
+
"",
|
|
17620
17881
|
"## Runtime Gate",
|
|
17621
17882
|
"",
|
|
17622
17883
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -17627,6 +17888,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
17627
17888
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
17628
17889
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
17629
17890
|
"",
|
|
17891
|
+
"## Hosted Riddle Handoff",
|
|
17892
|
+
"",
|
|
17893
|
+
"```text",
|
|
17894
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
17895
|
+
"```",
|
|
17896
|
+
"",
|
|
17630
17897
|
"## OC Handoff Prompt",
|
|
17631
17898
|
"",
|
|
17632
17899
|
"```text",
|
|
@@ -17643,22 +17910,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
17643
17910
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
17644
17911
|
`);
|
|
17645
17912
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
17913
|
+
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
17914
|
+
`);
|
|
17646
17915
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
17647
17916
|
`);
|
|
17648
17917
|
}
|
|
17649
|
-
function runRegressionPackForCli(options) {
|
|
17918
|
+
async function runRegressionPackForCli(options) {
|
|
17650
17919
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
17651
17920
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
17921
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
17652
17922
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
17923
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
17653
17924
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
17654
17925
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
17655
17926
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
17927
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
17656
17928
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
17657
|
-
const
|
|
17929
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
17930
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
17658
17931
|
const result = {
|
|
17659
17932
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
17660
17933
|
ok,
|
|
17661
17934
|
local_core_validated: localCoreValidated,
|
|
17935
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
17662
17936
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17663
17937
|
pack_path: filePath,
|
|
17664
17938
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -17668,13 +17942,19 @@ function runRegressionPackForCli(options) {
|
|
|
17668
17942
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
17669
17943
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
17670
17944
|
local_core: localCore,
|
|
17945
|
+
hosted_riddle: hostedRiddle,
|
|
17671
17946
|
openclaw_live_suite: {
|
|
17672
17947
|
target: regressionPackRecord(liveSuite.target),
|
|
17673
17948
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
17674
17949
|
case_count: liveCases.length,
|
|
17675
17950
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
17676
17951
|
},
|
|
17677
|
-
|
|
17952
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
17953
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
17954
|
+
localCoreOk: localCoreValidated,
|
|
17955
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
17956
|
+
hostedRiddleRequested
|
|
17957
|
+
}),
|
|
17678
17958
|
output_dir: profileOutputDirOption(options) || null
|
|
17679
17959
|
};
|
|
17680
17960
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -21361,7 +21641,7 @@ async function main() {
|
|
|
21361
21641
|
if (command === "regression-pack") {
|
|
21362
21642
|
const action = positional[1] || "run";
|
|
21363
21643
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
21364
|
-
const result = runRegressionPackForCli(options);
|
|
21644
|
+
const result = await runRegressionPackForCli(options);
|
|
21365
21645
|
writeRegressionPackRunResult(result, options);
|
|
21366
21646
|
process.exitCode = result.ok ? 0 : 1;
|
|
21367
21647
|
return;
|
package/dist/cli.js
CHANGED
|
@@ -40,6 +40,204 @@
|
|
|
40
40
|
"no-diff-prod-audit-default-capture-pass"
|
|
41
41
|
]
|
|
42
42
|
},
|
|
43
|
+
"hosted_riddle_suite": {
|
|
44
|
+
"runner": "riddle",
|
|
45
|
+
"target": {
|
|
46
|
+
"url": "https://riddledc.com/"
|
|
47
|
+
},
|
|
48
|
+
"result_log_fields": [
|
|
49
|
+
"case_id",
|
|
50
|
+
"profile_name",
|
|
51
|
+
"profile_status",
|
|
52
|
+
"route",
|
|
53
|
+
"riddle.job_id",
|
|
54
|
+
"riddle.status",
|
|
55
|
+
"artifacts",
|
|
56
|
+
"environment_blocker"
|
|
57
|
+
],
|
|
58
|
+
"cases": [
|
|
59
|
+
{
|
|
60
|
+
"id": "hosted-home-to-proof-route-change-pass",
|
|
61
|
+
"intent": "Run the generic hosted Riddle profile runner against production home, click Proof, and prove terminal /proof/ before involving OC.",
|
|
62
|
+
"profile": {
|
|
63
|
+
"version": "riddle-proof.profile.v1",
|
|
64
|
+
"name": "riddle-site-hosted-home-to-proof",
|
|
65
|
+
"target": {
|
|
66
|
+
"route": "/",
|
|
67
|
+
"wait_for_selector": "main#main-content",
|
|
68
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
69
|
+
"setup_actions": [
|
|
70
|
+
{ "type": "click", "selector": "a[href*='/proof']", "after_ms": 750 }
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
"checks": [
|
|
74
|
+
{ "type": "route_loaded", "expected_path": "/proof/" },
|
|
75
|
+
{ "type": "selector_visible", "selector": "main#main-content" },
|
|
76
|
+
{ "type": "text_visible", "text": "Proof" }
|
|
77
|
+
],
|
|
78
|
+
"metadata": {
|
|
79
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
80
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"expect": { "profile_status": "passed" }
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "hosted-proof-to-home-route-change-pass",
|
|
87
|
+
"intent": "Run the generic hosted Riddle profile runner from /proof/, click the root/Home link, and prove terminal /.",
|
|
88
|
+
"profile": {
|
|
89
|
+
"version": "riddle-proof.profile.v1",
|
|
90
|
+
"name": "riddle-site-hosted-proof-to-home",
|
|
91
|
+
"target": {
|
|
92
|
+
"route": "/proof/",
|
|
93
|
+
"wait_for_selector": "main#main-content",
|
|
94
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
95
|
+
"setup_actions": [
|
|
96
|
+
{ "type": "click", "selector": "a[href='/'], a[href='https://riddledc.com/']", "after_ms": 750 }
|
|
97
|
+
]
|
|
98
|
+
},
|
|
99
|
+
"checks": [
|
|
100
|
+
{ "type": "route_loaded", "expected_path": "/" },
|
|
101
|
+
{ "type": "selector_visible", "selector": "main#main-content" }
|
|
102
|
+
],
|
|
103
|
+
"metadata": {
|
|
104
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
105
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
"expect": { "profile_status": "passed" }
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "hosted-pricing-query-hash-positive-pass",
|
|
112
|
+
"intent": "Run the hosted profile runner directly against Pricing with query/hash and prove slash, query, and hash are preserved.",
|
|
113
|
+
"profile": {
|
|
114
|
+
"version": "riddle-proof.profile.v1",
|
|
115
|
+
"name": "riddle-site-hosted-pricing-query-hash",
|
|
116
|
+
"target": {
|
|
117
|
+
"route": "/pricing/?rp_probe=1#pricing-probe",
|
|
118
|
+
"wait_for_selector": "main#main-content",
|
|
119
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
120
|
+
"setup_actions": [
|
|
121
|
+
{
|
|
122
|
+
"type": "window_eval",
|
|
123
|
+
"script": "return window.location.hash;",
|
|
124
|
+
"expect_return": "#pricing-probe",
|
|
125
|
+
"store_return_to": "__riddleProof.pricingHash",
|
|
126
|
+
"capture_return": true
|
|
127
|
+
}
|
|
128
|
+
]
|
|
129
|
+
},
|
|
130
|
+
"checks": [
|
|
131
|
+
{ "type": "route_loaded", "expected_path": "/pricing/" },
|
|
132
|
+
{ "type": "url_search_param_equals", "param": "rp_probe", "expected_value": "1" },
|
|
133
|
+
{ "type": "selector_visible", "selector": "main#main-content" }
|
|
134
|
+
],
|
|
135
|
+
"metadata": {
|
|
136
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
137
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
"expect": { "profile_status": "passed" }
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"id": "hosted-pricing-query-hash-dropped-blocker",
|
|
144
|
+
"intent": "Run a hosted negative control where Pricing lacks the expected query/hash and require a product_regression profile result.",
|
|
145
|
+
"profile": {
|
|
146
|
+
"version": "riddle-proof.profile.v1",
|
|
147
|
+
"name": "riddle-site-hosted-pricing-query-hash-dropped",
|
|
148
|
+
"target": {
|
|
149
|
+
"route": "/pricing/",
|
|
150
|
+
"wait_for_selector": "main#main-content",
|
|
151
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
152
|
+
},
|
|
153
|
+
"checks": [
|
|
154
|
+
{ "type": "route_loaded", "expected_path": "/pricing/" },
|
|
155
|
+
{ "type": "url_search_param_equals", "param": "rp_probe", "expected_value": "1" }
|
|
156
|
+
],
|
|
157
|
+
"metadata": {
|
|
158
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
159
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
"expect": { "profile_status": "product_regression" }
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "hosted-no-diff-prod-audit-pass",
|
|
166
|
+
"intent": "Run a hosted no-diff production audit profile with no implementation or OC wrapper involved.",
|
|
167
|
+
"profile": {
|
|
168
|
+
"version": "riddle-proof.profile.v1",
|
|
169
|
+
"name": "riddle-site-hosted-no-diff-prod-audit",
|
|
170
|
+
"target": {
|
|
171
|
+
"route": "/",
|
|
172
|
+
"wait_for_selector": "main#main-content",
|
|
173
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
174
|
+
},
|
|
175
|
+
"checks": [
|
|
176
|
+
{ "type": "route_loaded", "expected_path": "/" },
|
|
177
|
+
{ "type": "selector_visible", "selector": "main#main-content" },
|
|
178
|
+
{ "type": "no_fatal_console_errors" }
|
|
179
|
+
],
|
|
180
|
+
"metadata": {
|
|
181
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
182
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
"expect": { "profile_status": "passed" }
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
"id": "hosted-missing-selector-timeout-blocker",
|
|
189
|
+
"intent": "Run a hosted profile with a missing selector and require a clean product_regression result, not a lifecycle failure.",
|
|
190
|
+
"profile": {
|
|
191
|
+
"version": "riddle-proof.profile.v1",
|
|
192
|
+
"name": "riddle-site-hosted-missing-selector",
|
|
193
|
+
"target": {
|
|
194
|
+
"route": "/",
|
|
195
|
+
"wait_for_selector": "main#main-content",
|
|
196
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }]
|
|
197
|
+
},
|
|
198
|
+
"checks": [
|
|
199
|
+
{ "type": "selector_visible", "selector": "#riddle-proof-missing-selector-timeout-smoke-20260604" }
|
|
200
|
+
],
|
|
201
|
+
"metadata": {
|
|
202
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
203
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
"expect": { "profile_status": "product_regression" }
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
"id": "hosted-thrown-error-specific-blocker",
|
|
210
|
+
"intent": "Run a hosted profile whose setup action throws a marker and require the marker to surface as a product_regression result.",
|
|
211
|
+
"profile": {
|
|
212
|
+
"version": "riddle-proof.profile.v1",
|
|
213
|
+
"name": "riddle-site-hosted-thrown-error",
|
|
214
|
+
"target": {
|
|
215
|
+
"route": "/",
|
|
216
|
+
"wait_for_selector": "main#main-content",
|
|
217
|
+
"viewports": [{ "name": "desktop", "width": 1280, "height": 900 }],
|
|
218
|
+
"setup_actions": [
|
|
219
|
+
{
|
|
220
|
+
"type": "window_eval",
|
|
221
|
+
"script": "throw new Error('RIDDLE_PROOF_HOSTED_THROWN_ERROR_SMOKE_20260604');",
|
|
222
|
+
"timeout_ms": 1000
|
|
223
|
+
}
|
|
224
|
+
]
|
|
225
|
+
},
|
|
226
|
+
"checks": [
|
|
227
|
+
{ "type": "route_loaded", "expected_path": "/" }
|
|
228
|
+
],
|
|
229
|
+
"metadata": {
|
|
230
|
+
"pack_id": "riddle_proof_hosted_flow",
|
|
231
|
+
"pack_public_name": "Riddle Proof Hosted Flow"
|
|
232
|
+
}
|
|
233
|
+
},
|
|
234
|
+
"expect": {
|
|
235
|
+
"profile_status": "product_regression",
|
|
236
|
+
"message_contains": "RIDDLE_PROOF_HOSTED_THROWN_ERROR_SMOKE_20260604"
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
]
|
|
240
|
+
},
|
|
43
241
|
"openclaw_live_suite": {
|
|
44
242
|
"target": {
|
|
45
243
|
"repo": "davisdiehl/riddle-site",
|