@riddledc/riddle-proof 0.8.34 → 0.8.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/advanced/index.d.cts +1 -1
- package/dist/advanced/index.d.ts +1 -1
- package/dist/advanced/proof-run-engine.d.cts +1 -1
- package/dist/advanced/proof-run-engine.d.ts +1 -1
- package/dist/{chunk-U73JPBZW.js → chunk-E25K5PDM.js} +287 -7
- package/dist/{chunk-PEWAIEER.js → chunk-Z2LCVROU.js} +126 -0
- package/dist/cli/index.js +2 -2
- package/dist/cli.cjs +412 -6
- package/dist/cli.js +2 -2
- package/dist/index.cjs +126 -0
- package/dist/index.js +1 -1
- package/dist/profile/index.cjs +126 -0
- package/dist/profile/index.js +1 -1
- package/dist/profile.cjs +126 -0
- package/dist/profile.d.cts +2 -0
- package/dist/profile.d.ts +2 -0
- package/dist/profile.js +1 -1
- package/dist/{proof-run-engine-BqaeqAze.d.ts → proof-run-engine-BmNYuOJ7.d.ts} +3 -3
- package/dist/{proof-run-engine-4dM37pEx.d.cts → proof-run-engine-DYUu2mqY.d.cts} +3 -3
- package/dist/proof-run-engine.d.cts +1 -1
- package/dist/proof-run-engine.d.ts +1 -1
- package/examples/regression-packs/oc-flow-regression.json +212 -0
- package/package.json +1 -1
package/dist/cli.cjs
CHANGED
|
@@ -9147,6 +9147,8 @@ function normalizeSetupAction(input, index) {
|
|
|
9147
9147
|
expect_changed: booleanValue(valueFromOwn(input, "expect_changed", "expectChanged", "should_change", "shouldChange", "changed")),
|
|
9148
9148
|
until_path: untilPath,
|
|
9149
9149
|
until_expected_value: hasUntilExpectedValue ? toJsonValue(valueFromOwn(input, "until_expected_value", "untilExpectedValue", "until_expected", "untilExpected", "until_value", "untilValue", "expected_value", "expectedValue", "expected")) : void 0,
|
|
9150
|
+
expected_path: stringFromOwn(input, "expected_path", "expectedPath", "expected_terminal_path", "expectedTerminalPath"),
|
|
9151
|
+
expected_url: stringFromOwn(input, "expected_url", "expectedUrl", "expected_terminal_url", "expectedTerminalUrl"),
|
|
9150
9152
|
max_calls: maxCalls,
|
|
9151
9153
|
tap_burst_size: tapBurstSize,
|
|
9152
9154
|
settle_ms: settleMs,
|
|
@@ -11488,6 +11490,80 @@ function routePathMatches(observed, expected, targetUrl) {
|
|
|
11488
11490
|
if (normalizedObserved === normalizedExpected) return true;
|
|
11489
11491
|
return normalizedObserved === normalizeRoutePath(mountedExpectedRoutePath(targetUrl, expected));
|
|
11490
11492
|
}
|
|
11493
|
+
function setupActionExpectedRoute(action) {
|
|
11494
|
+
const expectedUrl = typeof action.expected_url === "string" && action.expected_url.trim()
|
|
11495
|
+
? action.expected_url.trim()
|
|
11496
|
+
: typeof action.expectedUrl === "string" && action.expectedUrl.trim()
|
|
11497
|
+
? action.expectedUrl.trim()
|
|
11498
|
+
: "";
|
|
11499
|
+
const expectedPath = typeof action.expected_path === "string" && action.expected_path.trim()
|
|
11500
|
+
? action.expected_path.trim()
|
|
11501
|
+
: typeof action.expectedPath === "string" && action.expectedPath.trim()
|
|
11502
|
+
? action.expectedPath.trim()
|
|
11503
|
+
: "";
|
|
11504
|
+
if (!expectedUrl && !expectedPath) return null;
|
|
11505
|
+
return { expected_url: expectedUrl || undefined, expected_path: expectedPath || undefined };
|
|
11506
|
+
}
|
|
11507
|
+
function setupUrlMatchesExpectedRoute(href, expected) {
|
|
11508
|
+
if (!expected) return true;
|
|
11509
|
+
let observedUrl;
|
|
11510
|
+
try {
|
|
11511
|
+
observedUrl = new URL(href, targetUrl);
|
|
11512
|
+
} catch {
|
|
11513
|
+
return false;
|
|
11514
|
+
}
|
|
11515
|
+
if (expected.expected_url) {
|
|
11516
|
+
let expectedUrl;
|
|
11517
|
+
try {
|
|
11518
|
+
expectedUrl = new URL(expected.expected_url, targetUrl);
|
|
11519
|
+
} catch {
|
|
11520
|
+
return false;
|
|
11521
|
+
}
|
|
11522
|
+
return observedUrl.href === expectedUrl.href;
|
|
11523
|
+
}
|
|
11524
|
+
const expectedPath = expected.expected_path || "/";
|
|
11525
|
+
if (/[?#]/.test(expectedPath)) {
|
|
11526
|
+
const observedRoute = observedUrl.pathname + observedUrl.search + observedUrl.hash;
|
|
11527
|
+
const normalizedObservedRoute = observedRoute === "/" ? "/" : observedRoute.replace(/\/+(?=[?#]|$)/, "");
|
|
11528
|
+
const normalizedExpectedRoute = expectedPath === "/" ? "/" : expectedPath.replace(/\/+(?=[?#]|$)/, "");
|
|
11529
|
+
return normalizedObservedRoute === normalizedExpectedRoute;
|
|
11530
|
+
}
|
|
11531
|
+
return routePathMatches(observedUrl.pathname, expectedPath, targetUrl);
|
|
11532
|
+
}
|
|
11533
|
+
function setupObservedRouteEvidence(expected, waitError) {
|
|
11534
|
+
let observedUrl = page.url();
|
|
11535
|
+
let observedPath = "";
|
|
11536
|
+
let observedRoute = "";
|
|
11537
|
+
try {
|
|
11538
|
+
const url = new URL(observedUrl, targetUrl);
|
|
11539
|
+
observedUrl = url.href;
|
|
11540
|
+
observedPath = url.pathname;
|
|
11541
|
+
observedRoute = url.pathname + url.search + url.hash;
|
|
11542
|
+
} catch {
|
|
11543
|
+
observedPath = "";
|
|
11544
|
+
observedRoute = "";
|
|
11545
|
+
}
|
|
11546
|
+
return {
|
|
11547
|
+
expected_url: expected && expected.expected_url || undefined,
|
|
11548
|
+
expected_path: expected && expected.expected_path || undefined,
|
|
11549
|
+
observed_url: observedUrl,
|
|
11550
|
+
observed_path: observedPath,
|
|
11551
|
+
observed_route: observedRoute,
|
|
11552
|
+
route_matched: setupUrlMatchesExpectedRoute(observedUrl, expected),
|
|
11553
|
+
route_wait_error: waitError ? String(waitError && waitError.message ? waitError.message : waitError).slice(0, 1000) : undefined,
|
|
11554
|
+
};
|
|
11555
|
+
}
|
|
11556
|
+
async function waitForSetupActionRoute(action, timeout) {
|
|
11557
|
+
const expected = setupActionExpectedRoute(action);
|
|
11558
|
+
if (!expected) return null;
|
|
11559
|
+
let waitError;
|
|
11560
|
+
try {
|
|
11561
|
+
await page.waitForURL((url) => setupUrlMatchesExpectedRoute(url.href, expected), { timeout: Math.min(timeout, 20000) });
|
|
11562
|
+
} catch (error) {
|
|
11563
|
+
waitError = error;
|
|
11564
|
+
}
|
|
11565
|
+
return setupObservedRouteEvidence(expected, waitError);
|
|
11566
|
+
}
|
|
11491
11567
|
function routeOk(route, targetUrl) {
|
|
11492
11568
|
return Boolean(route && (route.matched || routePathMatches(route.observed, route.expected_path, targetUrl)) && !route.error && (route.http_status == null || route.http_status < 400));
|
|
11493
11569
|
}
|
|
@@ -14381,11 +14457,22 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
14381
14457
|
const prepared = await resolveSetupTapTarget(action, base, scope, timeout);
|
|
14382
14458
|
if (prepared.result) return prepared.result;
|
|
14383
14459
|
await dispatchSetupTapPoint(prepared.target.point, prepared.target.pointerType, prepared.target.durationMs);
|
|
14460
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
14461
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
14462
|
+
return {
|
|
14463
|
+
...base,
|
|
14464
|
+
...setupScopeEvidence(scope),
|
|
14465
|
+
...setupTapTargetEvidence(prepared.target),
|
|
14466
|
+
...routeEvidence,
|
|
14467
|
+
reason: "expected_route_not_reached",
|
|
14468
|
+
};
|
|
14469
|
+
}
|
|
14384
14470
|
return {
|
|
14385
14471
|
...base,
|
|
14386
14472
|
...setupScopeEvidence(scope),
|
|
14387
14473
|
ok: true,
|
|
14388
14474
|
...setupTapTargetEvidence(prepared.target),
|
|
14475
|
+
...routeEvidence,
|
|
14389
14476
|
};
|
|
14390
14477
|
}
|
|
14391
14478
|
if (type === "tap_until") {
|
|
@@ -15246,6 +15333,26 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
15246
15333
|
: { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
15247
15334
|
if (clickCount > 1) await page.mouse.click(fallbackPoint.x, fallbackPoint.y, { clickCount });
|
|
15248
15335
|
else await page.mouse.click(fallbackPoint.x, fallbackPoint.y);
|
|
15336
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
15337
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
15338
|
+
return {
|
|
15339
|
+
...base,
|
|
15340
|
+
...setupScopeEvidence(scope),
|
|
15341
|
+
count,
|
|
15342
|
+
target_index: targetIndex,
|
|
15343
|
+
text: matchedText,
|
|
15344
|
+
force: action.force === true || undefined,
|
|
15345
|
+
fallback_to_tap: true,
|
|
15346
|
+
input_dispatch: "playwright_mouse",
|
|
15347
|
+
click_error: String(error && error.message ? error.message : error).slice(0, 1000),
|
|
15348
|
+
click_count: clickCount > 1 ? clickCount : undefined,
|
|
15349
|
+
coordinate_mode: mode,
|
|
15350
|
+
x: position ? fromX : undefined,
|
|
15351
|
+
y: position ? fromY : undefined,
|
|
15352
|
+
...routeEvidence,
|
|
15353
|
+
reason: "expected_route_not_reached",
|
|
15354
|
+
};
|
|
15355
|
+
}
|
|
15249
15356
|
return {
|
|
15250
15357
|
...base,
|
|
15251
15358
|
...setupScopeEvidence(scope),
|
|
@@ -15261,6 +15368,24 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
15261
15368
|
coordinate_mode: mode,
|
|
15262
15369
|
x: position ? fromX : undefined,
|
|
15263
15370
|
y: position ? fromY : undefined,
|
|
15371
|
+
...routeEvidence,
|
|
15372
|
+
};
|
|
15373
|
+
}
|
|
15374
|
+
const routeEvidence = await waitForSetupActionRoute(action, timeout);
|
|
15375
|
+
if (routeEvidence && !routeEvidence.route_matched) {
|
|
15376
|
+
return {
|
|
15377
|
+
...base,
|
|
15378
|
+
...setupScopeEvidence(scope),
|
|
15379
|
+
count,
|
|
15380
|
+
target_index: targetIndex,
|
|
15381
|
+
text: matchedText,
|
|
15382
|
+
force: action.force === true || undefined,
|
|
15383
|
+
click_count: clickCount > 1 ? clickCount : undefined,
|
|
15384
|
+
coordinate_mode: mode,
|
|
15385
|
+
x: position ? fromX : undefined,
|
|
15386
|
+
y: position ? fromY : undefined,
|
|
15387
|
+
...routeEvidence,
|
|
15388
|
+
reason: "expected_route_not_reached",
|
|
15264
15389
|
};
|
|
15265
15390
|
}
|
|
15266
15391
|
return {
|
|
@@ -15275,6 +15400,7 @@ async function executeSetupAction(action, ordinal, viewport) {
|
|
|
15275
15400
|
coordinate_mode: mode,
|
|
15276
15401
|
x: position ? fromX : undefined,
|
|
15277
15402
|
y: position ? fromY : undefined,
|
|
15403
|
+
...routeEvidence,
|
|
15278
15404
|
};
|
|
15279
15405
|
}
|
|
15280
15406
|
if (type === "fill" || type === "set_input_value") {
|
|
@@ -17157,6 +17283,7 @@ var KNOWN_CLI_OPTIONS = /* @__PURE__ */ new Set([
|
|
|
17157
17283
|
"format",
|
|
17158
17284
|
"framework",
|
|
17159
17285
|
"help",
|
|
17286
|
+
"hostedRiddle",
|
|
17160
17287
|
"image",
|
|
17161
17288
|
"input",
|
|
17162
17289
|
"inputDir",
|
|
@@ -17228,7 +17355,7 @@ function usage() {
|
|
|
17228
17355
|
" riddle-proof-loop run-profile --profile <file|json|-> --url <base-url> [--base-url <base-url>] [--runner riddle] [--viewport-name <name[,name...]>] [--strict true|false; default false] [--split-viewports true|false; default false] [--balance-preflight true|false; default true] [--poll-attempts n] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json] [--quiet]",
|
|
17229
17356
|
" riddle-proof-loop run-profile aggregate --profile <file|json|-> --url <base-url> [--base-url <base-url>] --input-dir <dir>|--inputs <path[,path...]> [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17230
17357
|
" riddle-proof-loop run-profile recover --profile <file|json|-> --url <base-url> [--base-url <base-url>] --job <job-id> [--viewport-name <name[,name...]>] [--output <dir>|--output-dir <dir>] [--result-format json|compact-json|summary|none; default json]",
|
|
17231
|
-
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17358
|
+
" riddle-proof-loop regression-pack run [--pack oc-flow-regression|--pack-file <file>] [--local-core true|false; default true] [--hosted-riddle true|false; default false] [--format json|markdown|compact-json; default json] [--output <dir>|--output-dir <dir>]",
|
|
17232
17359
|
" riddle-proof-loop profile-body-assertions --artifact <file|url|-> --candidates-json <file|json|-> [--required-json <file|json|->] [--format json|body-contains]",
|
|
17233
17360
|
" riddle-proof-loop profile-http-status-preflight --profile <file|json|-> --url <base-url> [--format json|summary]",
|
|
17234
17361
|
" riddle-proof-loop riddle-preview-deploy <build-dir> <label> [--framework spa|static]",
|
|
@@ -17539,6 +17666,244 @@ function runRegressionPackLocalCore(pack) {
|
|
|
17539
17666
|
stdout_tail: ok ? void 0 : tailLines(child.stdout || "")
|
|
17540
17667
|
};
|
|
17541
17668
|
}
|
|
17669
|
+
function regressionPackHostedRiddleRequested(options) {
|
|
17670
|
+
return optionBoolean(options, "hostedRiddle") ?? false;
|
|
17671
|
+
}
|
|
17672
|
+
function regressionPackHostedRiddleSuite(pack) {
|
|
17673
|
+
return regressionPackRecord(pack.hosted_riddle_suite);
|
|
17674
|
+
}
|
|
17675
|
+
function regressionPackHostedRiddleCases(pack) {
|
|
17676
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17677
|
+
return Array.isArray(suite.cases) ? suite.cases.map(regressionPackRecord) : [];
|
|
17678
|
+
}
|
|
17679
|
+
function regressionPackHostedRiddleExpectedStatus(testCase) {
|
|
17680
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17681
|
+
return cliString(expect.profile_status) || cliString(expect.status);
|
|
17682
|
+
}
|
|
17683
|
+
function regressionPackHostedRiddleExpectedMessage(testCase) {
|
|
17684
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17685
|
+
return cliString(expect.message_contains);
|
|
17686
|
+
}
|
|
17687
|
+
function regressionPackHostedRiddlePlan(pack) {
|
|
17688
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17689
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17690
|
+
return {
|
|
17691
|
+
requested: false,
|
|
17692
|
+
configured: cases.length > 0,
|
|
17693
|
+
ok: true,
|
|
17694
|
+
runner: cliString(suite.runner) || "riddle",
|
|
17695
|
+
target: regressionPackRecord(suite.target),
|
|
17696
|
+
case_count: cases.length,
|
|
17697
|
+
case_ids: cases.map((item) => cliString(item.id)).filter(Boolean)
|
|
17698
|
+
};
|
|
17699
|
+
}
|
|
17700
|
+
function regressionPackHostedRiddleCaseOutputDir(outputDir, caseId) {
|
|
17701
|
+
return outputDir ? import_node_path6.default.join(outputDir, "hosted-riddle", safeProfileOutputSegment(caseId)) : void 0;
|
|
17702
|
+
}
|
|
17703
|
+
function compactHostedRiddleCaseResult(testCase, result, ok) {
|
|
17704
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17705
|
+
return {
|
|
17706
|
+
id: cliString(testCase.id) || result.profile_name,
|
|
17707
|
+
intent: cliString(testCase.intent) || null,
|
|
17708
|
+
ok,
|
|
17709
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17710
|
+
expected_message_contains: expectedMessage || null,
|
|
17711
|
+
expected_message_found: expectedMessage ? JSON.stringify(result).includes(expectedMessage) : void 0,
|
|
17712
|
+
status: result.status,
|
|
17713
|
+
profile_name: result.profile_name,
|
|
17714
|
+
summary: result.summary,
|
|
17715
|
+
route: result.route,
|
|
17716
|
+
riddle: result.riddle,
|
|
17717
|
+
artifacts: result.artifacts,
|
|
17718
|
+
environment_blocker: result.environment_blocker,
|
|
17719
|
+
error: result.error
|
|
17720
|
+
};
|
|
17721
|
+
}
|
|
17722
|
+
function hostedRiddleBlockedCaseResult(testCase, error, environmentBlocker) {
|
|
17723
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17724
|
+
return {
|
|
17725
|
+
id: cliString(testCase.id) || cliString(rawProfile.name) || "hosted-riddle-case",
|
|
17726
|
+
intent: cliString(testCase.intent) || null,
|
|
17727
|
+
ok: false,
|
|
17728
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17729
|
+
expected_message_contains: regressionPackHostedRiddleExpectedMessage(testCase) || null,
|
|
17730
|
+
status: "environment_blocked",
|
|
17731
|
+
profile_name: cliString(rawProfile.name) || null,
|
|
17732
|
+
summary: error,
|
|
17733
|
+
environment_blocker: environmentBlocker,
|
|
17734
|
+
error
|
|
17735
|
+
};
|
|
17736
|
+
}
|
|
17737
|
+
async function hostedRiddleSuiteEnvironmentBlocker(cases, options) {
|
|
17738
|
+
if (!runProfileBalancePreflightOption(options) || !cases.length) return void 0;
|
|
17739
|
+
const client = createRiddleApiClient(riddleClientConfig(options));
|
|
17740
|
+
const requiredSeconds = cases.length * RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB;
|
|
17741
|
+
let balance;
|
|
17742
|
+
try {
|
|
17743
|
+
balance = await client.getBalance();
|
|
17744
|
+
} catch (error) {
|
|
17745
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17746
|
+
return {
|
|
17747
|
+
error: message,
|
|
17748
|
+
environment_blocker: {
|
|
17749
|
+
source: "riddle_api",
|
|
17750
|
+
endpoint: "/v1/balance",
|
|
17751
|
+
reason: "balance_preflight_failed",
|
|
17752
|
+
balance_preflight: true,
|
|
17753
|
+
job_count: cases.length,
|
|
17754
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17755
|
+
required_seconds: requiredSeconds,
|
|
17756
|
+
...riddleApiErrorBlockerMetadata(error),
|
|
17757
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17758
|
+
}
|
|
17759
|
+
};
|
|
17760
|
+
}
|
|
17761
|
+
const availableSeconds = riddleBalanceAvailableSeconds(balance);
|
|
17762
|
+
if (availableSeconds === void 0 || availableSeconds >= requiredSeconds) return void 0;
|
|
17763
|
+
const reservedSeconds = cliFiniteNumber(balance.reserved_seconds);
|
|
17764
|
+
const totalSeconds = cliFiniteNumber(balance.total_seconds);
|
|
17765
|
+
const holdsCount = cliFiniteNumber(balance.holds_count);
|
|
17766
|
+
return {
|
|
17767
|
+
error: `Riddle hosted regression balance preflight failed: ${availableSeconds}s available for ${cases.length} serial hosted job(s), minimum ${requiredSeconds}s required.`,
|
|
17768
|
+
environment_blocker: {
|
|
17769
|
+
source: "riddle_api",
|
|
17770
|
+
endpoint: "/v1/balance",
|
|
17771
|
+
reason: "insufficient_balance",
|
|
17772
|
+
error: "Insufficient available balance",
|
|
17773
|
+
balance_preflight: true,
|
|
17774
|
+
job_count: cases.length,
|
|
17775
|
+
seconds_per_job: RIDDLE_PROFILE_BALANCE_PREFLIGHT_MIN_SECONDS_PER_JOB,
|
|
17776
|
+
required_seconds: requiredSeconds,
|
|
17777
|
+
available_seconds: availableSeconds,
|
|
17778
|
+
deficit_seconds: requiredSeconds - availableSeconds,
|
|
17779
|
+
...reservedSeconds === void 0 ? {} : { reserved_seconds: reservedSeconds },
|
|
17780
|
+
...totalSeconds === void 0 ? {} : { total_seconds: totalSeconds },
|
|
17781
|
+
...holdsCount === void 0 ? {} : { holds_count: holdsCount },
|
|
17782
|
+
...apiKeySourceBlockerMetadata(client)
|
|
17783
|
+
}
|
|
17784
|
+
};
|
|
17785
|
+
}
|
|
17786
|
+
async function runRegressionPackHostedRiddle(pack, options) {
|
|
17787
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17788
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17789
|
+
const runner = cliString(suite.runner) || "riddle";
|
|
17790
|
+
const target = regressionPackRecord(suite.target);
|
|
17791
|
+
const baseUrl = cliString(target.url) || cliString(target.base_url) || cliString(target.baseUrl) || optionString(options, "url") || optionString(options, "baseUrl");
|
|
17792
|
+
const outputDir = profileOutputDirOption(options);
|
|
17793
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17794
|
+
const results = [];
|
|
17795
|
+
const suiteBlocker = await hostedRiddleSuiteEnvironmentBlocker(cases, options);
|
|
17796
|
+
if (suiteBlocker) {
|
|
17797
|
+
const blockedCases = cases.map((testCase) => hostedRiddleBlockedCaseResult(testCase, suiteBlocker.error, suiteBlocker.environment_blocker));
|
|
17798
|
+
return {
|
|
17799
|
+
requested: true,
|
|
17800
|
+
configured: cases.length > 0,
|
|
17801
|
+
ok: false,
|
|
17802
|
+
runner,
|
|
17803
|
+
target,
|
|
17804
|
+
started_at: startedAt,
|
|
17805
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17806
|
+
case_count: cases.length,
|
|
17807
|
+
passed_case_count: 0,
|
|
17808
|
+
failed_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17809
|
+
environment_blocked_cases: blockedCases.map((item) => cliString(item.id)).filter((item) => Boolean(item)),
|
|
17810
|
+
environment_blocker: suiteBlocker.environment_blocker,
|
|
17811
|
+
error: suiteBlocker.error,
|
|
17812
|
+
cases: blockedCases
|
|
17813
|
+
};
|
|
17814
|
+
}
|
|
17815
|
+
for (const testCase of cases) {
|
|
17816
|
+
const caseId = cliString(testCase.id) || `case-${results.length + 1}`;
|
|
17817
|
+
const rawProfile = regressionPackRecord(testCase.profile);
|
|
17818
|
+
if (!Object.keys(rawProfile).length) {
|
|
17819
|
+
results.push({
|
|
17820
|
+
id: caseId,
|
|
17821
|
+
intent: cliString(testCase.intent) || null,
|
|
17822
|
+
ok: false,
|
|
17823
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17824
|
+
status: "configuration_error",
|
|
17825
|
+
error: "hosted_riddle_suite case requires profile."
|
|
17826
|
+
});
|
|
17827
|
+
continue;
|
|
17828
|
+
}
|
|
17829
|
+
let result;
|
|
17830
|
+
const caseOutputDir = regressionPackHostedRiddleCaseOutputDir(outputDir, caseId);
|
|
17831
|
+
const caseOptions = {
|
|
17832
|
+
...options,
|
|
17833
|
+
runner,
|
|
17834
|
+
...caseOutputDir ? { output: caseOutputDir, outputDir: caseOutputDir } : {}
|
|
17835
|
+
};
|
|
17836
|
+
try {
|
|
17837
|
+
const profile = profileWithSelectedViewportNamesForCli(
|
|
17838
|
+
normalizeRiddleProofProfile(rawProfile, { url: baseUrl }),
|
|
17839
|
+
options
|
|
17840
|
+
);
|
|
17841
|
+
result = await runProfileForCli(profile, caseOptions);
|
|
17842
|
+
writeProfileOutput(caseOutputDir, result);
|
|
17843
|
+
} catch (error) {
|
|
17844
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
17845
|
+
results.push({
|
|
17846
|
+
id: caseId,
|
|
17847
|
+
intent: cliString(testCase.intent) || null,
|
|
17848
|
+
ok: false,
|
|
17849
|
+
expected_status: regressionPackHostedRiddleExpectedStatus(testCase) || null,
|
|
17850
|
+
status: "configuration_error",
|
|
17851
|
+
error: message,
|
|
17852
|
+
output_dir: caseOutputDir || null
|
|
17853
|
+
});
|
|
17854
|
+
continue;
|
|
17855
|
+
}
|
|
17856
|
+
const expectedStatus = regressionPackHostedRiddleExpectedStatus(testCase);
|
|
17857
|
+
const expectedMessage = regressionPackHostedRiddleExpectedMessage(testCase);
|
|
17858
|
+
const statusOk = expectedStatus ? result.status === expectedStatus : result.status === "passed";
|
|
17859
|
+
const messageOk = expectedMessage ? JSON.stringify(result).includes(expectedMessage) : true;
|
|
17860
|
+
const ok = statusOk && messageOk;
|
|
17861
|
+
results.push({
|
|
17862
|
+
...compactHostedRiddleCaseResult(testCase, result, ok),
|
|
17863
|
+
output_dir: caseOutputDir || null
|
|
17864
|
+
});
|
|
17865
|
+
}
|
|
17866
|
+
const failedCases = results.filter((item) => item.ok !== true).map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17867
|
+
const environmentBlockedCases = results.filter((item) => cliString(item.status) === "environment_blocked").map((item) => cliString(item.id)).filter((item) => Boolean(item));
|
|
17868
|
+
return {
|
|
17869
|
+
requested: true,
|
|
17870
|
+
configured: cases.length > 0,
|
|
17871
|
+
ok: cases.length > 0 && !failedCases.length,
|
|
17872
|
+
runner,
|
|
17873
|
+
target,
|
|
17874
|
+
started_at: startedAt,
|
|
17875
|
+
finished_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17876
|
+
case_count: cases.length,
|
|
17877
|
+
passed_case_count: results.length - failedCases.length,
|
|
17878
|
+
failed_cases: failedCases,
|
|
17879
|
+
environment_blocked_cases: environmentBlockedCases,
|
|
17880
|
+
cases: results
|
|
17881
|
+
};
|
|
17882
|
+
}
|
|
17883
|
+
function hostedRiddleHandoffPromptForRegressionPack(pack) {
|
|
17884
|
+
const cases = regressionPackHostedRiddleCases(pack);
|
|
17885
|
+
const suite = regressionPackHostedRiddleSuite(pack);
|
|
17886
|
+
const target = regressionPackRecord(suite.target);
|
|
17887
|
+
const lines = [
|
|
17888
|
+
"Run the hosted Riddle generic regression suite before involving OpenClaw.",
|
|
17889
|
+
"",
|
|
17890
|
+
"Command:",
|
|
17891
|
+
"riddle-proof-loop regression-pack run --pack oc-flow-regression --local-core true --hosted-riddle true --format markdown --output-dir artifacts/riddle-proof/hosted-regression",
|
|
17892
|
+
"",
|
|
17893
|
+
"Target:",
|
|
17894
|
+
...Object.entries(target).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`),
|
|
17895
|
+
"",
|
|
17896
|
+
"Cases:",
|
|
17897
|
+
...cases.map((testCase, index) => {
|
|
17898
|
+
const expect = regressionPackRecord(testCase.expect);
|
|
17899
|
+
return `${index + 1}. ${cliString(testCase.id) || "unnamed-case"}: ${cliString(testCase.intent) || "no intent"}
|
|
17900
|
+
Expect: ${JSON.stringify(expect)}`;
|
|
17901
|
+
}),
|
|
17902
|
+
"",
|
|
17903
|
+
"Only pass the batch to OC after local_core.ok and hosted_riddle.ok are both true."
|
|
17904
|
+
].filter((line) => line !== "");
|
|
17905
|
+
return lines.join("\n");
|
|
17906
|
+
}
|
|
17542
17907
|
function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
17543
17908
|
const minimumVersions = regressionPackRecord(pack.minimum_versions);
|
|
17544
17909
|
const runtimeGate = regressionPackRecord(pack.runtime_gate);
|
|
@@ -17560,6 +17925,7 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17560
17925
|
"- Run cases serially, not as one broad parallel batch.",
|
|
17561
17926
|
"- If loaded metadata is stale, stop and restart/reload the gateway before counting results.",
|
|
17562
17927
|
"- If any generic lifecycle marker appears, report the exact marker and artifact, then stop the counted batch.",
|
|
17928
|
+
input.localCoreOk ? input.hostedRiddleOk ? "- Generic local core and hosted Riddle suites are green; OC should only validate wrapper/runtime adapter behavior." : input.hostedRiddleRequested ? "- Hosted Riddle generic suite did not pass; do not treat OC failures as wrapper-only until it is green." : "- Hosted Riddle generic suite was not run in this regression-pack invocation; run it before counting OC as the late adapter gate." : "- Local generic core suite is not green or was not run; keep OC as blocked-on-generic until local core is green.",
|
|
17563
17929
|
`- Forbidden terminal markers: ${forbiddenMarkers.join(", ") || "none"}.`,
|
|
17564
17930
|
fields.length ? `- Log fields for every counted run: ${fields.join(", ")}.` : "",
|
|
17565
17931
|
"",
|
|
@@ -17572,12 +17938,13 @@ function openClawHandoffPromptForRegressionPack(pack, input) {
|
|
|
17572
17938
|
].join("\n");
|
|
17573
17939
|
}),
|
|
17574
17940
|
"",
|
|
17575
|
-
input.localCoreOk ? "Local generic core suite is green
|
|
17941
|
+
input.localCoreOk ? "Local generic core suite is green." : "Local generic core suite is not green or was not run; do not count OC failures as wrapper-only until local core is green."
|
|
17576
17942
|
].filter((line) => line !== "");
|
|
17577
17943
|
return lines.join("\n");
|
|
17578
17944
|
}
|
|
17579
17945
|
function compactRegressionPackRunResult(result) {
|
|
17580
17946
|
const localCore = regressionPackRecord(result.local_core);
|
|
17947
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17581
17948
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
17582
17949
|
return {
|
|
17583
17950
|
version: result.version,
|
|
@@ -17592,12 +17959,22 @@ function compactRegressionPackRunResult(result) {
|
|
|
17592
17959
|
failed_cases: localCore.failed_cases,
|
|
17593
17960
|
forbidden_terminal_markers_seen: localCore.forbidden_terminal_markers_seen
|
|
17594
17961
|
},
|
|
17962
|
+
hosted_riddle: {
|
|
17963
|
+
requested: hostedRiddle.requested,
|
|
17964
|
+
configured: hostedRiddle.configured,
|
|
17965
|
+
ok: hostedRiddle.ok,
|
|
17966
|
+
runner: hostedRiddle.runner,
|
|
17967
|
+
case_count: hostedRiddle.case_count,
|
|
17968
|
+
failed_cases: hostedRiddle.failed_cases,
|
|
17969
|
+
environment_blocked_cases: hostedRiddle.environment_blocked_cases
|
|
17970
|
+
},
|
|
17595
17971
|
openclaw_live_case_count: openClaw.case_count,
|
|
17596
17972
|
output_dir: result.output_dir
|
|
17597
17973
|
};
|
|
17598
17974
|
}
|
|
17599
17975
|
function regressionPackRunMarkdown(result) {
|
|
17600
17976
|
const localCore = regressionPackRecord(result.local_core);
|
|
17977
|
+
const hostedRiddle = regressionPackRecord(result.hosted_riddle);
|
|
17601
17978
|
const runtimeGate = regressionPackRecord(result.runtime_gate);
|
|
17602
17979
|
const minimumVersions = regressionPackRecord(result.minimum_versions);
|
|
17603
17980
|
const openClaw = regressionPackRecord(result.openclaw_live_suite);
|
|
@@ -17617,6 +17994,16 @@ function regressionPackRunMarkdown(result) {
|
|
|
17617
17994
|
`- failed cases: ${regressionPackStringArray(localCore.failed_cases).join(", ") || "none"}`,
|
|
17618
17995
|
`- forbidden markers seen: ${regressionPackStringArray(localCore.forbidden_terminal_markers_seen).join(", ") || "none"}`,
|
|
17619
17996
|
"",
|
|
17997
|
+
"## Hosted Riddle",
|
|
17998
|
+
"",
|
|
17999
|
+
`- requested: ${hostedRiddle.requested === true}`,
|
|
18000
|
+
`- configured: ${hostedRiddle.configured === true}`,
|
|
18001
|
+
`- ok: ${hostedRiddle.ok === true}`,
|
|
18002
|
+
`- runner: ${cliString(hostedRiddle.runner) || "n/a"}`,
|
|
18003
|
+
`- cases: ${hostedRiddle.case_count ?? "n/a"}`,
|
|
18004
|
+
`- failed cases: ${regressionPackStringArray(hostedRiddle.failed_cases).join(", ") || "none"}`,
|
|
18005
|
+
`- environment blocked cases: ${regressionPackStringArray(hostedRiddle.environment_blocked_cases).join(", ") || "none"}`,
|
|
18006
|
+
"",
|
|
17620
18007
|
"## Runtime Gate",
|
|
17621
18008
|
"",
|
|
17622
18009
|
`- tool: ${cliString(runtimeGate.tool) || "n/a"}`,
|
|
@@ -17627,6 +18014,12 @@ function regressionPackRunMarkdown(result) {
|
|
|
17627
18014
|
`- case count: ${openClaw.case_count ?? "n/a"}`,
|
|
17628
18015
|
`- result log fields: ${regressionPackStringArray(openClaw.result_log_fields).join(", ") || "n/a"}`,
|
|
17629
18016
|
"",
|
|
18017
|
+
"## Hosted Riddle Handoff",
|
|
18018
|
+
"",
|
|
18019
|
+
"```text",
|
|
18020
|
+
cliString(result.hosted_riddle_handoff_prompt) || "",
|
|
18021
|
+
"```",
|
|
18022
|
+
"",
|
|
17630
18023
|
"## OC Handoff Prompt",
|
|
17631
18024
|
"",
|
|
17632
18025
|
"```text",
|
|
@@ -17643,22 +18036,29 @@ function writeRegressionPackOutput(outputDir, result) {
|
|
|
17643
18036
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "regression-pack-result.json"), `${JSON.stringify(result, null, 2)}
|
|
17644
18037
|
`);
|
|
17645
18038
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "summary.md"), regressionPackRunMarkdown(result));
|
|
18039
|
+
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "hosted-riddle-handoff.md"), `${cliString(result.hosted_riddle_handoff_prompt) || ""}
|
|
18040
|
+
`);
|
|
17646
18041
|
(0, import_node_fs6.writeFileSync)(import_node_path6.default.join(outputDir, "oc-handoff.md"), `${cliString(result.openclaw_handoff_prompt) || ""}
|
|
17647
18042
|
`);
|
|
17648
18043
|
}
|
|
17649
|
-
function runRegressionPackForCli(options) {
|
|
18044
|
+
async function runRegressionPackForCli(options) {
|
|
17650
18045
|
const { filePath, pack } = readRegressionPackForCli(options);
|
|
17651
18046
|
const localCoreRequested = optionBoolean(options, "localCore") ?? true;
|
|
18047
|
+
const hostedRiddleRequested = regressionPackHostedRiddleRequested(options);
|
|
17652
18048
|
const localCore = localCoreRequested ? runRegressionPackLocalCore(pack) : { requested: false, ok: true, command: regressionPackCommandForLocalCore(pack) };
|
|
18049
|
+
const hostedRiddle = hostedRiddleRequested ? await runRegressionPackHostedRiddle(pack, options) : regressionPackHostedRiddlePlan(pack);
|
|
17653
18050
|
const liveSuite = regressionPackRecord(pack.openclaw_live_suite);
|
|
17654
18051
|
const liveCases = Array.isArray(liveSuite.cases) ? liveSuite.cases : [];
|
|
17655
18052
|
const localCoreRecord = regressionPackRecord(localCore);
|
|
18053
|
+
const hostedRiddleRecord = regressionPackRecord(hostedRiddle);
|
|
17656
18054
|
const localCoreValidated = localCoreRecord.requested === true && localCoreRecord.ok === true;
|
|
17657
|
-
const
|
|
18055
|
+
const hostedRiddleValidated = hostedRiddleRecord.requested === true && hostedRiddleRecord.ok === true;
|
|
18056
|
+
const ok = (localCoreRequested ? localCoreValidated : true) && (hostedRiddleRequested ? hostedRiddleValidated : true);
|
|
17658
18057
|
const result = {
|
|
17659
18058
|
version: "riddle-proof.regression-pack-run-result.v1",
|
|
17660
18059
|
ok,
|
|
17661
18060
|
local_core_validated: localCoreValidated,
|
|
18061
|
+
hosted_riddle_validated: hostedRiddleValidated,
|
|
17662
18062
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
17663
18063
|
pack_path: filePath,
|
|
17664
18064
|
pack_id: cliString(pack.pack_id) || null,
|
|
@@ -17668,13 +18068,19 @@ function runRegressionPackForCli(options) {
|
|
|
17668
18068
|
runtime_gate: regressionPackRecord(pack.runtime_gate),
|
|
17669
18069
|
forbidden_terminal_markers: regressionPackStringArray(pack.forbidden_terminal_markers),
|
|
17670
18070
|
local_core: localCore,
|
|
18071
|
+
hosted_riddle: hostedRiddle,
|
|
17671
18072
|
openclaw_live_suite: {
|
|
17672
18073
|
target: regressionPackRecord(liveSuite.target),
|
|
17673
18074
|
result_log_fields: regressionPackStringArray(liveSuite.result_log_fields),
|
|
17674
18075
|
case_count: liveCases.length,
|
|
17675
18076
|
case_ids: liveCases.map((item) => cliString(regressionPackRecord(item).id)).filter(Boolean)
|
|
17676
18077
|
},
|
|
17677
|
-
|
|
18078
|
+
hosted_riddle_handoff_prompt: hostedRiddleHandoffPromptForRegressionPack(pack),
|
|
18079
|
+
openclaw_handoff_prompt: openClawHandoffPromptForRegressionPack(pack, {
|
|
18080
|
+
localCoreOk: localCoreValidated,
|
|
18081
|
+
hostedRiddleOk: hostedRiddleValidated,
|
|
18082
|
+
hostedRiddleRequested
|
|
18083
|
+
}),
|
|
17678
18084
|
output_dir: profileOutputDirOption(options) || null
|
|
17679
18085
|
};
|
|
17680
18086
|
writeRegressionPackOutput(profileOutputDirOption(options), result);
|
|
@@ -21361,7 +21767,7 @@ async function main() {
|
|
|
21361
21767
|
if (command === "regression-pack") {
|
|
21362
21768
|
const action = positional[1] || "run";
|
|
21363
21769
|
if (action !== "run") throw new Error("Only `regression-pack run` is supported.");
|
|
21364
|
-
const result = runRegressionPackForCli(options);
|
|
21770
|
+
const result = await runRegressionPackForCli(options);
|
|
21365
21771
|
writeRegressionPackRunResult(result, options);
|
|
21366
21772
|
process.exitCode = result.ok ? 0 : 1;
|
|
21367
21773
|
return;
|
package/dist/cli.js
CHANGED