@riddledc/riddle-proof 0.8.34 → 0.8.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/advanced/index.d.cts +1 -1
- package/dist/advanced/index.d.ts +1 -1
- package/dist/advanced/proof-run-engine.d.cts +1 -1
- package/dist/advanced/proof-run-engine.d.ts +1 -1
- package/dist/{chunk-U73JPBZW.js → chunk-E25K5PDM.js} +287 -7
- package/dist/{chunk-PEWAIEER.js → chunk-Z2LCVROU.js} +126 -0
- package/dist/cli/index.js +2 -2
- package/dist/cli.cjs +412 -6
- package/dist/cli.js +2 -2
- package/dist/index.cjs +126 -0
- package/dist/index.js +1 -1
- package/dist/profile/index.cjs +126 -0
- package/dist/profile/index.js +1 -1
- package/dist/profile.cjs +126 -0
- package/dist/profile.d.cts +2 -0
- package/dist/profile.d.ts +2 -0
- package/dist/profile.js +1 -1
- package/dist/{proof-run-engine-BqaeqAze.d.ts → proof-run-engine-BmNYuOJ7.d.ts} +3 -3
- package/dist/{proof-run-engine-4dM37pEx.d.cts → proof-run-engine-DYUu2mqY.d.cts} +3 -3
- package/dist/proof-run-engine.d.cts +1 -1
- package/dist/proof-run-engine.d.ts +1 -1
- package/examples/regression-packs/oc-flow-regression.json +212 -0
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { b as runner } from '../runner-4LJ5z0D-.cjs';
|
|
2
2
|
export { l as engineHarness } from '../engine-harness-LBfqbFSe.cjs';
|
|
3
3
|
export { p as proofRunCore } from '../proof-run-core-B1GeqkR8.cjs';
|
|
4
|
-
export { p as proofRunEngine } from '../proof-run-engine-
|
|
4
|
+
export { p as proofRunEngine } from '../proof-run-engine-DYUu2mqY.cjs';
|
|
5
5
|
import '../types.cjs';
|
package/dist/advanced/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { b as runner } from '../runner-BdQpOkZD.js';
|
|
2
2
|
export { l as engineHarness } from '../engine-harness-CMACHP6A.js';
|
|
3
3
|
export { p as proofRunCore } from '../proof-run-core-B1GeqkR8.js';
|
|
4
|
-
export { p as proofRunEngine } from '../proof-run-engine-
|
|
4
|
+
export { p as proofRunEngine } from '../proof-run-engine-BmNYuOJ7.js';
|
|
5
5
|
import '../types.js';
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { R as RiddleProofEngine, c as createRiddleProofEngine, e as executeWorkflow } from '../proof-run-engine-
|
|
1
|
+
export { R as RiddleProofEngine, c as createRiddleProofEngine, e as executeWorkflow } from '../proof-run-engine-DYUu2mqY.cjs';
|
|
2
2
|
import '../proof-run-core-B1GeqkR8.cjs';
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { R as RiddleProofEngine, c as createRiddleProofEngine, e as executeWorkflow } from '../proof-run-engine-
|
|
1
|
+
export { R as RiddleProofEngine, c as createRiddleProofEngine, e as executeWorkflow } from '../proof-run-engine-BmNYuOJ7.js';
|
|
2
2
|
import '../proof-run-core-B1GeqkR8.js';
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
profileStatusExitCode,
|
|
13
13
|
resolveRiddleProofProfileTargetUrl,
|
|
14
14
|
resolveRiddleProofProfileTimeoutSec
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-Z2LCVROU.js";
|
|
16
16
|
import {
|
|
17
17
|
createRiddleApiClient,
|
|
18
18
|
isTerminalRiddleJobStatus,
|
|
@@ -65,6 +65,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
65
65
|
"format",
|
|
66
66
|
"framework",
|
|
67
67
|
"help",
|
|
68
|
+
"hostedRiddle",
|
|
68
69
|
"image",
|
|
69
70
|
"input",
|
|
70
71
|
"inputDir",
|
|
@@ -136,7 +137,7 @@ function usage() {
|
|
|
136
137
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
137
138
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
138
139
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
139
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
140
141
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
141
142
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
142
143
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -447,6 +448,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
447
448
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
448
449
|
};
|
|
449
450
|
}
|
|
451
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
452
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
453
|
+
}
|
|
454
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
455
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
456
|
+
}
|
|
457
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
458
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
459
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
460
|
+
}
|
|
461
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
462
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
463
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
464
|
+
}
|
|
465
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
466
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
467
|
+
return cliString(expect.message_contains);
|
|
468
|
+
}
|
|
469
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
470
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
471
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
472
|
+
return {
|
|
473
|
+
requested: false,
|
|
474
|
+
configured: cases.length > 0,
|
|
475
|
+
ok: true,
|
|
476
|
+
runner: cliString(suite.runner) || "riddle",
|
|
477
|
+
target: regressionPackRecord(suite.target),
|
|
478
|
+
case_count: cases.length,
|
|
479
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
483
|
+
return outputDir ? path.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
484
|
+
}
|
|
485
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
486
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
487
|
+
return {
|
|
488
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
489
|
+
intent: cliString(testCase.intent) || null,
|
|
490
|
+
ok,
|
|
491
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
492
|
+
expected_message_contains: expectedMessage || null,
|
|
493
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
494
|
+
status: result.status,
|
|
495
|
+
profile_name: result.profile_name,
|
|
496
|
+
summary: result.summary,
|
|
497
|
+
route: result.route,
|
|
498
|
+
riddle: result.riddle,
|
|
499
|
+
artifacts: result.artifacts,
|
|
500
|
+
environment_blocker: result.environment_blocker,
|
|
501
|
+
error: result.error
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
505
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
506
|
+
return {
|
|
507
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
508
|
+
intent: cliString(testCase.intent) || null,
|
|
509
|
+
ok: false,
|
|
510
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
511
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
512
|
+
status: "environment_blocked",
|
|
513
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
514
|
+
summary: error,
|
|
515
|
+
environment_blocker: environmentBlocker,
|
|
516
|
+
error
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
520
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
521
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
522
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
523
|
+
let balance;
|
|
524
|
+
try {
|
|
525
|
+
balance = await client.getBalance();
|
|
526
|
+
} catch (error) {
|
|
527
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
528
|
+
return {
|
|
529
|
+
error: message,
|
|
530
|
+
environment_blocker: {
|
|
531
|
+
source: "riddle_api",
|
|
532
|
+
endpoint: "/v1/balance",
|
|
533
|
+
reason: "balance_preflight_failed",
|
|
534
|
+
balance_preflight: true,
|
|
535
|
+
job_count: cases.length,
|
|
536
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
537
|
+
required_seconds: requiredSeconds,
|
|
538
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
539
|
+
...apiKeySourceBlockerMetadata(client)
|
|
540
|
+
}
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
544
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
545
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
546
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
547
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
548
|
+
return {
|
|
549
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
550
|
+
environment_blocker: {
|
|
551
|
+
source: "riddle_api",
|
|
552
|
+
endpoint: "/v1/balance",
|
|
553
|
+
reason: "insufficient_balance",
|
|
554
|
+
error: "Insufficient available balance",
|
|
555
|
+
balance_preflight: true,
|
|
556
|
+
job_count: cases.length,
|
|
557
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
558
|
+
required_seconds: requiredSeconds,
|
|
559
|
+
available_seconds: availableSeconds,
|
|
560
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
561
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
562
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
563
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
564
|
+
...apiKeySourceBlockerMetadata(client)
|
|
565
|
+
}
|
|
566
|
+
};
|
|
567
|
+
}
|
|
568
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
569
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
570
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
571
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
572
|
+
const target = regressionPackRecord(suite.target);
|
|
573
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
574
|
+
const outputDir = profileOutputDirOption(options);
|
|
575
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
576
|
+
const results = [];
|
|
577
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
578
|
+
if (suiteBlocker) {
|
|
579
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
580
|
+
return {
|
|
581
|
+
requested: true,
|
|
582
|
+
configured: cases.length > 0,
|
|
583
|
+
ok: false,
|
|
584
|
+
runner,
|
|
585
|
+
target,
|
|
586
|
+
started_at: startedAt,
|
|
587
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
588
|
+
case_count: cases.length,
|
|
589
|
+
passed_case_count: 0,
|
|
590
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
591
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
592
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
593
|
+
error: suiteBlocker.error,
|
|
594
|
+
cases: blockedCases
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
for (const testCase of cases) {
|
|
598
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
599
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
600
|
+
if (!Object.keys(rawProfile).length) {
|
|
601
|
+
results.push({
|
|
602
|
+
id: caseId,
|
|
603
|
+
intent: cliString(testCase.intent) || null,
|
|
604
|
+
ok: false,
|
|
605
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
606
|
+
status: "configuration_error",
|
|
607
|
+
error: "hosted_riddle_suite case requires profile."
|
|
608
|
+
});
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
let result;
|
|
612
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
613
|
+
const caseOptions = {
|
|
614
|
+
...options,
|
|
615
|
+
runner,
|
|
616
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
617
|
+
};
|
|
618
|
+
try {
|
|
619
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
620
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
621
|
+
options
|
|
622
|
+
);
|
|
623
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
624
|
+
writeProfileOutput(caseOutputDir, result);
|
|
625
|
+
} catch (error) {
|
|
626
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
627
|
+
results.push({
|
|
628
|
+
id: caseId,
|
|
629
|
+
intent: cliString(testCase.intent) || null,
|
|
630
|
+
ok: false,
|
|
631
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
632
|
+
status: "configuration_error",
|
|
633
|
+
error: message,
|
|
634
|
+
output_dir: caseOutputDir || null
|
|
635
|
+
});
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
639
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
640
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
641
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
642
|
+
const ok = statusOk && messageOk;
|
|
643
|
+
results.push({
|
|
644
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
645
|
+
output_dir: caseOutputDir || null
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
649
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
650
|
+
return {
|
|
651
|
+
requested: true,
|
|
652
|
+
configured: cases.length > 0,
|
|
653
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
654
|
+
runner,
|
|
655
|
+
target,
|
|
656
|
+
started_at: startedAt,
|
|
657
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
658
|
+
case_count: cases.length,
|
|
659
|
+
passed_case_count: results.length - failedCases.length,
|
|
660
|
+
failed_cases: failedCases,
|
|
661
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
662
|
+
cases: results
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
666
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
667
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
668
|
+
const target = regressionPackRecord(suite.target);
|
|
669
|
+
const lines = [
|
|
670
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
671
|
+
"",
|
|
672
|
+
"Command:",
|
|
673
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
674
|
+
"",
|
|
675
|
+
"Target:",
|
|
676
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
677
|
+
"",
|
|
678
|
+
"Cases:",
|
|
679
|
+
...cases.map((testCase, index) => {
|
|
680
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
681
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
682
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
683
|
+
}),
|
|
684
|
+
"",
|
|
685
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
686
|
+
].filter((line) => line !== "");
|
|
687
|
+
return lines.join("\n");
|
|
688
|
+
}
|
|
450
689
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
451
690
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
452
691
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -468,6 +707,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
468
707
|
"- Run cases serially, not as one broad parallel batch.",
|
|
469
708
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
470
709
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
710
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
471
711
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
472
712
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
473
713
|
"",
|
|
@@ -480,12 +720,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
480
720
|
].join("\n");
|
|
481
721
|
}),
|
|
482
722
|
"",
|
|
483
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
723
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
484
724
|
].filter((line) => line !== "");
|
|
485
725
|
return lines.join("\n");
|
|
486
726
|
}
|
|
487
727
|
function compactRegressionPackRunResult(result) {
|
|
488
728
|
const localCore = regressionPackRecord(result.local_core);
|
|
729
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
489
730
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
490
731
|
return {
|
|
491
732
|
version: result.version,
|
|
@@ -500,12 +741,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
500
741
|
failed_cases: localCore.failed_cases,
|
|
501
742
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
502
743
|
},
|
|
744
|
+
hosted_riddle: {
|
|
745
|
+
requested: hostedRiddle.requested,
|
|
746
|
+
configured: hostedRiddle.configured,
|
|
747
|
+
ok: hostedRiddle.ok,
|
|
748
|
+
runner: hostedRiddle.runner,
|
|
749
|
+
case_count: hostedRiddle.case_count,
|
|
750
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
751
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
752
|
+
},
|
|
503
753
|
openclaw_live_case_count: openClaw.case_count,
|
|
504
754
|
output_dir: result.output_dir
|
|
505
755
|
};
|
|
506
756
|
}
|
|
507
757
|
function regressionPackRunMarkdown(result) {
|
|
508
758
|
const localCore = regressionPackRecord(result.local_core);
|
|
759
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
509
760
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
510
761
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
511
762
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -525,6 +776,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
525
776
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
526
777
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
527
778
|
"",
|
|
779
|
+
"## Hosted Riddle",
|
|
780
|
+
"",
|
|
781
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
782
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
783
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
784
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
785
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
786
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
787
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
788
|
+
"",
|
|
528
789
|
"## Runtime Gate",
|
|
529
790
|
"",
|
|
530
791
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -535,6 +796,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
535
796
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
536
797
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
537
798
|
"",
|
|
799
|
+
"## Hosted Riddle Handoff",
|
|
800
|
+
"",
|
|
801
|
+
"```text",
|
|
802
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
803
|
+
"```",
|
|
804
|
+
"",
|
|
538
805
|
"## OC Handoff Prompt",
|
|
539
806
|
"",
|
|
540
807
|
"```text",
|
|
@@ -551,22 +818,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
551
818
|
writeFileSync(path.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
552
819
|
`);
|
|
553
820
|
writeFileSync(path.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
821
|
+
writeFileSync(path.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
822
|
+
`);
|
|
554
823
|
writeFileSync(path.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
555
824
|
`);
|
|
556
825
|
}
|
|
557
|
-
function runRegressionPackForCli(options) {
|
|
826
|
+
async function runRegressionPackForCli(options) {
|
|
558
827
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
559
828
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
829
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
560
830
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
831
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
561
832
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
562
833
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
563
834
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
835
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
564
836
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
565
|
-
const
|
|
837
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
838
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
566
839
|
const result = {
|
|
567
840
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
568
841
|
ok,
|
|
569
842
|
local_core_validated: localCoreValidated,
|
|
843
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
570
844
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
571
845
|
pack_path: filePath,
|
|
572
846
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -576,13 +850,19 @@ function runRegressionPackForCli(options) {
|
|
|
576
850
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
577
851
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
578
852
|
local_core: localCore,
|
|
853
|
+
hosted_riddle: hostedRiddle,
|
|
579
854
|
openclaw_live_suite: {
|
|
580
855
|
target: regressionPackRecord(liveSuite.target),
|
|
581
856
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
582
857
|
case_count: liveCases.length,
|
|
583
858
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
584
859
|
},
|
|
585
|
-
|
|
860
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
861
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
862
|
+
localCoreOk: localCoreValidated,
|
|
863
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
864
|
+
hostedRiddleRequested
|
|
865
|
+
}),
|
|
586
866
|
output_dir: profileOutputDirOption(options) || null
|
|
587
867
|
};
|
|
588
868
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -4269,7 +4549,7 @@ async function main() {
|
|
|
4269
4549
|
if (command === "regression-pack") {
|
|
4270
4550
|
const action = positional[1] || "run";
|
|
4271
4551
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
4272
|
-
const result = runRegressionPackForCli(options);
|
|
4552
|
+
const result = await runRegressionPackForCli(options);
|
|
4273
4553
|
writeRegressionPackRunResult(result, options);
|
|
4274
4554
|
process.exitCode = result.ok ? 0 : 1;
|
|
4275
4555
|
return;
|
|
@@ -1331,6 +1331,8 @@ function normalizeSetupAction(input, index) {
|
|
|
1331
1331
|
expect_changed: booleanValue(valueFromOwn(input, "expect_changed", "expectChanged", "should_change", "shouldChange", "changed")),
|
|
1332
1332
|
until_path: untilPath,
|
|
1333
1333
|
until_expected_value: hasUntilExpectedValue ? toJsonValue(valueFromOwn(input, "until_expected_value", "untilExpectedValue", "until_expected", "untilExpected", "until_value", "untilValue", "expected_value", "expectedValue", "expected")) : void 0,
|
|
1334
|
+
expected_path: stringFromOwn(input, "expected_path", "expectedPath", "expected_terminal_path", "expectedTerminalPath"),
|
|
1335
|
+
expected_url: stringFromOwn(input, "expected_url", "expectedUrl", "expected_terminal_url", "expectedTerminalUrl"),
|
|
1334
1336
|
max_calls: maxCalls,
|
|
1335
1337
|
tap_burst_size: tapBurstSize,
|
|
1336
1338
|
settle_ms: settleMs,
|
|
@@ -3688,6 +3690,80 @@ function routePathMatches(observed, expected, targetUrl) {
|
|
|
3688
3690
|
if (normalizedObserved === normalizedExpected) return true;
|
|
3689
3691
|
return normalizedObserved === normalizeRoutePath(mountedExpectedRoutePath(targetUrl, expected));
|
|
3690
3692
|
}
|
|
3693
|
+
function setupActionExpectedRoute(action) {
|
|
3694
|
+
const expectedUrl = typeof action.expected_url === "string" && action.expected_url.trim()
|
|
3695
|
+
? action.expected_url.trim()
|
|
3696
|
+
: typeof action.expectedUrl === "string" && action.expectedUrl.trim()
|
|
3697
|
+
? action.expectedUrl.trim()
|
|
3698
|
+
: "";
|
|
3699
|
+
const expectedPath = typeof action.expected_path === "string" && action.expected_path.trim()
|
|
3700
|
+
? action.expected_path.trim()
|
|
3701
|
+
: typeof action.expectedPath === "string" && action.expectedPath.trim()
|
|
3702
|
+
? action.expectedPath.trim()
|
|
3703
|
+
: "";
|
|
3704
|
+
if (!expectedUrl && !expectedPath) return null;
|
|
3705
|
+
return { expected_url: expectedUrl || undefined, expected_path: expectedPath || undefined };
|
|
3706
|
+
}
|
|
3707
|
+
function setupUrlMatchesExpectedRoute(href, expected) {
|
|
3708
|
+
if (!expected) return true;
|
|
3709
|
+
let observedUrl;
|
|
3710
|
+
try {
|
|
3711
|
+
observedUrl = new URL(href, targetUrl);
|
|
3712
|
+
} catch {
|
|
3713
|
+
return false;
|
|
3714
|
+
}
|
|
3715
|
+
if (expected.expected_url) {
|
|
3716
|
+
let expectedUrl;
|
|
3717
|
+
try {
|
|
3718
|
+
expectedUrl = new URL(expected.expected_url, targetUrl);
|
|
3719
|
+
} catch {
|
|
3720
|
+
return false;
|
|
3721
|
+
}
|
|
3722
|
+
return observedUrl.href === expectedUrl.href;
|
|
3723
|
+
}
|
|
3724
|
+
const expectedPath = expected.expected_path || "/";
|
|
3725
|
+
if (/[?#]/.test(expectedPath)) {
|
|
3726
|
+
const observedRoute = observedUrl.pathname + observedUrl.search + observedUrl.hash;
|
|
3727
|
+
const normalizedObservedRoute = observedRoute === "/" ? "/" : observedRoute.replace(/\/+(?=[?#]|$)/, "");
|
|
3728
|
+
const normalizedExpectedRoute = expectedPath === "/" ? "/" : expectedPath.replace(/\/+(?=[?#]|$)/, "");
|
|
3729
|
+
return normalizedObservedRoute === normalizedExpectedRoute;
|
|
3730
|
+
}
|
|
3731
|
+
return routePathMatches(observedUrl.pathname, expectedPath, targetUrl);
|
|
3732
|
+
}
|
|
3733
|
+
function setupObservedRouteEvidence(expected, waitError) {
|
|
3734
|
+
let observedUrl = page.url();
|
|
3735
|
+
let observedPath = "";
|
|
3736
|
+
let observedRoute = "";
|
|
3737
|
+
try {
|
|
3738
|
+
const url = new URL(observedUrl, targetUrl);
|
|
3739
|
+
observedUrl = url.href;
|
|
3740
|
+
observedPath = url.pathname;
|
|
3741
|
+
observedRoute = url.pathname + url.search + url.hash;
|
|
3742
|
+
} catch {
|
|
3743
|
+
observedPath = "";
|
|
3744
|
+
observedRoute = "";
|
|
3745
|
+
}
|
|
3746
|
+
return {
|
|
3747
|
+
expected_url: expected && expected.expected_url || undefined,
|
|
3748
|
+
expected_path: expected && expected.expected_path || undefined,
|
|
3749
|
+
observed_url: observedUrl,
|
|
3750
|
+
observed_path: observedPath,
|
|
3751
|
+
observed_route: observedRoute,
|
|
3752
|
+
route_matched: setupUrlMatchesExpectedRoute(observedUrl, expected),
|
|
3753
|
+
route_wait_error: waitError ? String(waitError && waitError.message ? waitError.message : waitError).slice(0, 1000) : undefined,
|
|
3754
|
+
};
|
|
3755
|
+
}
|
|
3756
|
+
async function waitForSetupActionRoute(action, timeout) {
|
|
3757
|
+
const expected = setupActionExpectedRoute(action);
|
|
3758
|
+
if (!expected) return null;
|
|
3759
|
+
let waitError;
|
|
3760
|
+
try {
|
|
3761
|
+
await page.waitForURL((url) => setupUrlMatchesExpectedRoute(url.href, expected), { timeout: Math.min(timeout, 20000) });
|
|
3762
|
+
} catch (error) {
|
|
3763
|
+
waitError = error;
|
|
3764
|
+
}
|
|
3765
|
+
return setupObservedRouteEvidence(expected, waitError);
|
|
3766
|
+
}
|
|
3691
3767
|
function routeOk(route, targetUrl) {
|
|
3692
3768
|
return Boolean(route && (route.matched || routePathMatches(route.observed, route.expected_path, targetUrl)) && !route.error && (route.http_status == null || route.http_status < 400));
|
|
3693
3769
|
}
|
|
@@ -6581,11 +6657,22 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
6581
6657
|
const prepared = await resolveSetupTapTarget(action, base, scope, timeout);
|
|
6582
6658
|
if (prepared.result) return prepared.result;
|
|
6583
6659
|
await dispatchSetupTapPoint(prepared.target.point, prepared.target.pointerType, prepared.target.durationMs);
|
|
6660
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
6661
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
6662
|
+
return {
|
|
6663
|
+
...base,
|
|
6664
|
+
...setupScopeEvidence(scope),
|
|
6665
|
+
...setupTapTargetEvidence(prepared.target),
|
|
6666
|
+
...routeEvidence,
|
|
6667
|
+
reason: "expected_route_not_reached",
|
|
6668
|
+
};
|
|
6669
|
+
}
|
|
6584
6670
|
return {
|
|
6585
6671
|
...base,
|
|
6586
6672
|
...setupScopeEvidence(scope),
|
|
6587
6673
|
ok: true,
|
|
6588
6674
|
...setupTapTargetEvidence(prepared.target),
|
|
6675
|
+
...routeEvidence,
|
|
6589
6676
|
};
|
|
6590
6677
|
}
|
|
6591
6678
|
if (type === "tap_until") {
|
|
@@ -7446,6 +7533,26 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
7446
7533
|
: { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
7447
7534
|
if (clickCount > 1) await page.mouse.click(fallbackPoint.x, fallbackPoint.y, { clickCount });
|
|
7448
7535
|
else await page.mouse.click(fallbackPoint.x, fallbackPoint.y);
|
|
7536
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
7537
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
7538
|
+
return {
|
|
7539
|
+
...base,
|
|
7540
|
+
...setupScopeEvidence(scope),
|
|
7541
|
+
count,
|
|
7542
|
+
target_index: targetIndex,
|
|
7543
|
+
text: matchedText,
|
|
7544
|
+
force: action.force === true || undefined,
|
|
7545
|
+
fallback_to_tap: true,
|
|
7546
|
+
input_dispatch: "playwright_mouse",
|
|
7547
|
+
click_error: String(error && error.message ? error.message : error).slice(0, 1000),
|
|
7548
|
+
click_count: clickCount > 1 ? clickCount : undefined,
|
|
7549
|
+
coordinate_mode: mode,
|
|
7550
|
+
x: position ? fromX : undefined,
|
|
7551
|
+
y: position ? fromY : undefined,
|
|
7552
|
+
...routeEvidence,
|
|
7553
|
+
reason: "expected_route_not_reached",
|
|
7554
|
+
};
|
|
7555
|
+
}
|
|
7449
7556
|
return {
|
|
7450
7557
|
...base,
|
|
7451
7558
|
...setupScopeEvidence(scope),
|
|
@@ -7461,6 +7568,24 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
7461
7568
|
coordinate_mode: mode,
|
|
7462
7569
|
x: position ? fromX : undefined,
|
|
7463
7570
|
y: position ? fromY : undefined,
|
|
7571
|
+
...routeEvidence,
|
|
7572
|
+
};
|
|
7573
|
+
}
|
|
7574
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
7575
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
7576
|
+
return {
|
|
7577
|
+
...base,
|
|
7578
|
+
...setupScopeEvidence(scope),
|
|
7579
|
+
count,
|
|
7580
|
+
target_index: targetIndex,
|
|
7581
|
+
text: matchedText,
|
|
7582
|
+
force: action.force === true || undefined,
|
|
7583
|
+
click_count: clickCount > 1 ? clickCount : undefined,
|
|
7584
|
+
coordinate_mode: mode,
|
|
7585
|
+
x: position ? fromX : undefined,
|
|
7586
|
+
y: position ? fromY : undefined,
|
|
7587
|
+
...routeEvidence,
|
|
7588
|
+
reason: "expected_route_not_reached",
|
|
7464
7589
|
};
|
|
7465
7590
|
}
|
|
7466
7591
|
return {
|
|
@@ -7475,6 +7600,7 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
7475
7600
|
coordinate_mode: mode,
|
|
7476
7601
|
x: position ? fromX : undefined,
|
|
7477
7602
|
y: position ? fromY : undefined,
|
|
7603
|
+
...routeEvidence,
|
|
7478
7604
|
};
|
|
7479
7605
|
}
|
|
7480
7606
|
if (type === "fill" || type === "set_input_value") {
|
package/dist/cli/index.js
CHANGED