@f-o-h/cli 0.1.73 → 0.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/foh.js +196 -32
- package/package.json +1 -1
package/dist/foh.js
CHANGED
|
@@ -14383,6 +14383,11 @@ async function runSetupCertifyLoop(agentId, params) {
|
|
|
14383
14383
|
}
|
|
14384
14384
|
|
|
14385
14385
|
// src/lib/agent-publish-gate.ts
|
|
14386
|
+
function normalizePublishCertificationEvidence(publish) {
|
|
14387
|
+
const evidence = publish.certification_evidence;
|
|
14388
|
+
if (!evidence || typeof evidence !== "object" || Array.isArray(evidence)) return null;
|
|
14389
|
+
return evidence;
|
|
14390
|
+
}
|
|
14386
14391
|
async function validateCertifyAndPublishAgent(opts) {
|
|
14387
14392
|
const validation = await apiFetch(
|
|
14388
14393
|
`/v1/console/agents/${opts.agentId}/validate`,
|
|
@@ -14403,11 +14408,14 @@ async function validateCertifyAndPublishAgent(opts) {
|
|
|
14403
14408
|
apiUrlOverride: opts.apiUrlOverride,
|
|
14404
14409
|
orgId: opts.orgId
|
|
14405
14410
|
});
|
|
14411
|
+
const evidence = normalizePublishCertificationEvidence(publish);
|
|
14406
14412
|
return {
|
|
14407
14413
|
validation,
|
|
14408
14414
|
certification: {
|
|
14409
14415
|
status: "not_run",
|
|
14410
|
-
reason_code: "publish_consumes_existing_certification_evidence"
|
|
14416
|
+
reason_code: "publish_consumes_existing_certification_evidence",
|
|
14417
|
+
evidence_source: evidence?.source ?? null,
|
|
14418
|
+
evidence
|
|
14411
14419
|
},
|
|
14412
14420
|
publish
|
|
14413
14421
|
};
|
|
@@ -32790,7 +32798,7 @@ var StdioServerTransport = class {
|
|
|
32790
32798
|
};
|
|
32791
32799
|
|
|
32792
32800
|
// src/lib/cli-version.ts
|
|
32793
|
-
var CLI_VERSION = "0.1.
|
|
32801
|
+
var CLI_VERSION = "0.1.75";
|
|
32794
32802
|
|
|
32795
32803
|
// src/commands/mcp-serve.ts
|
|
32796
32804
|
var DEFAULT_TIMEOUT_MS = 12e4;
|
|
@@ -35480,9 +35488,11 @@ function registerCertify(program3) {
|
|
|
35480
35488
|
const certify = program3.command("certify").description("Produce release certification evidence for an agent");
|
|
35481
35489
|
certify.command("run").description("Run certification for an exact agent draft/profile and emit release evidence").requiredOption("--agent <id>", "Agent ID to certify").option("--profile <profile>", "Certification profile: smoke, release, or stress", "release").option("--adaptive-runs <n>", "Adaptive runs override; release defaults to a budget-safe 5").option("--journeys <list>", "Comma-separated journey allowlist").option("--scenario-ids <list>", "Comma-separated scenario ID allowlist").option("--channel <channel>", "Channel filter: chat, voice, or mixed", "mixed").option("--out <path>", "Write certification run JSON to this file path").option("--api-url <url>", "API base URL override").option("--json", "Output as machine-readable JSON").action(async (opts) => {
|
|
35482
35490
|
try {
|
|
35491
|
+
const totalStarted = Date.now();
|
|
35483
35492
|
const profile = normalizeProfile(opts.profile);
|
|
35484
35493
|
const mode = modeForProfile(profile);
|
|
35485
35494
|
const adaptiveRuns = Math.max(1, Math.min(120, Number(opts.adaptiveRuns ?? defaultAdaptiveRuns(profile)) || defaultAdaptiveRuns(profile)));
|
|
35495
|
+
const apiStarted = Date.now();
|
|
35486
35496
|
const response = await apiFetch(
|
|
35487
35497
|
`/v1/console/agents/${opts.agent}/sim-certify`,
|
|
35488
35498
|
{
|
|
@@ -35497,6 +35507,7 @@ function registerCertify(program3) {
|
|
|
35497
35507
|
apiUrlOverride: opts.apiUrl
|
|
35498
35508
|
}
|
|
35499
35509
|
);
|
|
35510
|
+
const apiMs = Math.max(0, Date.now() - apiStarted);
|
|
35500
35511
|
if (!response.ok || !response.certificate) {
|
|
35501
35512
|
throw new FohError({
|
|
35502
35513
|
step: "certify.run",
|
|
@@ -35512,6 +35523,10 @@ function registerCertify(program3) {
|
|
|
35512
35523
|
reason_code: passed ? "certification_passed" : "certification_failed",
|
|
35513
35524
|
profile,
|
|
35514
35525
|
mode,
|
|
35526
|
+
timing: {
|
|
35527
|
+
api_ms: apiMs,
|
|
35528
|
+
total_ms: Math.max(0, Date.now() - totalStarted)
|
|
35529
|
+
},
|
|
35515
35530
|
certificate: response.certificate,
|
|
35516
35531
|
next_commands: passed ? [`foh agent publish --agent ${opts.agent} --json`] : [`foh certify run --agent ${opts.agent} --profile ${profile} --json`]
|
|
35517
35532
|
};
|
|
@@ -37507,8 +37522,10 @@ function readFreshCache(filePath, maxAgeMs) {
|
|
|
37507
37522
|
const payload = JSON.parse((0, import_node_fs3.readFileSync)(filePath, "utf8"));
|
|
37508
37523
|
const createdAt = Date.parse(String(payload.created_at || ""));
|
|
37509
37524
|
if (!Number.isFinite(createdAt)) return null;
|
|
37510
|
-
|
|
37511
|
-
|
|
37525
|
+
const ageMs = Date.now() - createdAt;
|
|
37526
|
+
if (ageMs > maxAgeMs) return null;
|
|
37527
|
+
if (payload.value === void 0) return null;
|
|
37528
|
+
return { value: payload.value, ageMs };
|
|
37512
37529
|
} catch {
|
|
37513
37530
|
return null;
|
|
37514
37531
|
}
|
|
@@ -37532,7 +37549,16 @@ async function withProofCache(options, run) {
|
|
|
37532
37549
|
if (!resolvedDir) {
|
|
37533
37550
|
return {
|
|
37534
37551
|
value: await run(),
|
|
37535
|
-
metadata: {
|
|
37552
|
+
metadata: {
|
|
37553
|
+
hit: false,
|
|
37554
|
+
miss: true,
|
|
37555
|
+
key: "disabled",
|
|
37556
|
+
cache_path: "",
|
|
37557
|
+
waited_ms: 0,
|
|
37558
|
+
waited_for_peer: false,
|
|
37559
|
+
source: "disabled",
|
|
37560
|
+
age_ms: null
|
|
37561
|
+
}
|
|
37536
37562
|
};
|
|
37537
37563
|
}
|
|
37538
37564
|
const key = cacheKey2(options.kind, options.keyParts);
|
|
@@ -37545,8 +37571,17 @@ async function withProofCache(options, run) {
|
|
|
37545
37571
|
const existing = readFreshCache(cachePath, maxAgeMs);
|
|
37546
37572
|
if (existing) {
|
|
37547
37573
|
return {
|
|
37548
|
-
value: existing,
|
|
37549
|
-
metadata: {
|
|
37574
|
+
value: existing.value,
|
|
37575
|
+
metadata: {
|
|
37576
|
+
hit: true,
|
|
37577
|
+
miss: false,
|
|
37578
|
+
key,
|
|
37579
|
+
cache_path: publicPath(cachePath),
|
|
37580
|
+
waited_ms: 0,
|
|
37581
|
+
waited_for_peer: false,
|
|
37582
|
+
source: "cache",
|
|
37583
|
+
age_ms: existing.ageMs
|
|
37584
|
+
}
|
|
37550
37585
|
};
|
|
37551
37586
|
}
|
|
37552
37587
|
let lockOwner = false;
|
|
@@ -37560,8 +37595,17 @@ async function withProofCache(options, run) {
|
|
|
37560
37595
|
const waitedValue = readFreshCache(cachePath, maxAgeMs);
|
|
37561
37596
|
if (waitedValue) {
|
|
37562
37597
|
return {
|
|
37563
|
-
value: waitedValue,
|
|
37564
|
-
metadata: {
|
|
37598
|
+
value: waitedValue.value,
|
|
37599
|
+
metadata: {
|
|
37600
|
+
hit: true,
|
|
37601
|
+
miss: false,
|
|
37602
|
+
key,
|
|
37603
|
+
cache_path: publicPath(cachePath),
|
|
37604
|
+
waited_ms: Date.now() - started,
|
|
37605
|
+
waited_for_peer: true,
|
|
37606
|
+
source: "cache",
|
|
37607
|
+
age_ms: waitedValue.ageMs
|
|
37608
|
+
}
|
|
37565
37609
|
};
|
|
37566
37610
|
}
|
|
37567
37611
|
}
|
|
@@ -37571,7 +37615,16 @@ async function withProofCache(options, run) {
|
|
|
37571
37615
|
writeCache(cachePath, value);
|
|
37572
37616
|
return {
|
|
37573
37617
|
value,
|
|
37574
|
-
metadata: {
|
|
37618
|
+
metadata: {
|
|
37619
|
+
hit: false,
|
|
37620
|
+
miss: true,
|
|
37621
|
+
key,
|
|
37622
|
+
cache_path: publicPath(cachePath),
|
|
37623
|
+
waited_ms: 0,
|
|
37624
|
+
waited_for_peer: false,
|
|
37625
|
+
source: "fresh_run",
|
|
37626
|
+
age_ms: null
|
|
37627
|
+
}
|
|
37575
37628
|
};
|
|
37576
37629
|
} finally {
|
|
37577
37630
|
if (lockOwner) (0, import_node_fs3.rmSync)(lockPath, { recursive: true, force: true });
|
|
@@ -37579,6 +37632,37 @@ async function withProofCache(options, run) {
|
|
|
37579
37632
|
}
|
|
37580
37633
|
|
|
37581
37634
|
// src/commands/prove.ts
|
|
37635
|
+
function addTiming(timings, name, startedMs) {
|
|
37636
|
+
timings.set(name, (timings.get(name) ?? 0) + Math.max(0, Date.now() - startedMs));
|
|
37637
|
+
}
|
|
37638
|
+
async function timedCheck(timings, name, fn) {
|
|
37639
|
+
const startedMs = Date.now();
|
|
37640
|
+
try {
|
|
37641
|
+
return await fn();
|
|
37642
|
+
} finally {
|
|
37643
|
+
addTiming(timings, name, startedMs);
|
|
37644
|
+
}
|
|
37645
|
+
}
|
|
37646
|
+
function enrichCheckTimings(checks, timings) {
|
|
37647
|
+
return checks.map((check2) => ({
|
|
37648
|
+
...check2,
|
|
37649
|
+
duration_ms: Math.max(0, timings.get(check2.name) ?? check2.duration_ms ?? 0)
|
|
37650
|
+
}));
|
|
37651
|
+
}
|
|
37652
|
+
function proofTimingSummary(checks, commandStartedMs) {
|
|
37653
|
+
const totalCheckDurationMs = checks.reduce((sum, check2) => sum + (check2.duration_ms ?? 0), 0);
|
|
37654
|
+
return {
|
|
37655
|
+
total_ms: Math.max(0, Date.now() - commandStartedMs),
|
|
37656
|
+
total_check_duration_ms: totalCheckDurationMs,
|
|
37657
|
+
slow_checks: checks.filter((check2) => (check2.duration_ms ?? 0) > 0).map((check2) => ({
|
|
37658
|
+
name: check2.name,
|
|
37659
|
+
category: check2.category,
|
|
37660
|
+
status: check2.status,
|
|
37661
|
+
reason_code: check2.reason_code,
|
|
37662
|
+
duration_ms: check2.duration_ms ?? 0
|
|
37663
|
+
})).sort((a, b) => b.duration_ms - a.duration_ms).slice(0, 10)
|
|
37664
|
+
};
|
|
37665
|
+
}
|
|
37582
37666
|
function categoryForCheck(name) {
|
|
37583
37667
|
if (name === "auth") return "auth";
|
|
37584
37668
|
if (name === "contact_channel" || name === "voice_realtime_health") return "voice";
|
|
@@ -37674,7 +37758,9 @@ function isProviderCapacityBlocked(onboarding) {
|
|
|
37674
37758
|
}
|
|
37675
37759
|
function registerProve(program3) {
|
|
37676
37760
|
program3.command("prove").description("Produce one setup/runtime proof bundle for an agent").option("--agent <id>", "Agent ID to prove").option("--org <id>", "Org ID (default: stored org from foh org use)").option("--include-certification", "Run explicit simulation certification check (slow)").option("--cert-mode <m>", "Simulation cert mode when --include-certification is set: quick, full, stress", "quick").option("--cert-adaptive-runs <n>", "Adaptive runs for full/stress certification when included", "30").option("--cert-max-improvement-rounds <n>", "Max prompt improvement rounds in included cert loop (0-5)", "1").option("--mission <mission>", "Proof mission: setup, widget, voice, publish", "setup").option("--contact-path <mode>", "Voice contact path: auto, managed, or byon", "auto").option("--mutation-mode <mode>", "Proof mutation mode: read-only or ensure", "read-only").option("--repair", "Alias for --mutation-mode ensure").option("--require-phone", "Hold proof if no phone/contact number is provisioned").option("--skip-cert", "Deprecated compatibility flag; certification is skipped unless --include-certification is set").option("--skip-smoke", "Skip widget runtime smoke check").option("--skip-voice-health", "Skip realtime voice provider health check").option("--proof-cache-dir <path>", "Optional local proof cache directory for shared certification results").option("--out <path>", "Write signed proof report JSON to this path").option("--strict", "Exit non-zero unless all non-skipped checks pass").option("--api-url <url>", "API base URL override").option("--json", "Output as JSON").action(async (opts) => withCommandErrorHandling(async () => {
|
|
37761
|
+
const commandStartedMs = Date.now();
|
|
37677
37762
|
const checks = [];
|
|
37763
|
+
const checkTimings = /* @__PURE__ */ new Map();
|
|
37678
37764
|
const mission = normalizeMission(opts.mission);
|
|
37679
37765
|
const contactPath = normalizeContactPath(opts.contactPath);
|
|
37680
37766
|
const mutationMode = normalizeMutationMode(opts.mutationMode, Boolean(opts.repair));
|
|
@@ -37685,7 +37771,9 @@ function registerProve(program3) {
|
|
|
37685
37771
|
correlationIds: []
|
|
37686
37772
|
};
|
|
37687
37773
|
try {
|
|
37774
|
+
const authStartedMs = Date.now();
|
|
37688
37775
|
const creds = loadCredentials(opts.apiUrl);
|
|
37776
|
+
addTiming(checkTimings, "auth", authStartedMs);
|
|
37689
37777
|
ctx.apiUrl = creds.apiUrl;
|
|
37690
37778
|
ctx.tokenPresent = Boolean(creds.token);
|
|
37691
37779
|
ctx.orgId = opts.org || creds.orgId;
|
|
@@ -37694,13 +37782,14 @@ function registerProve(program3) {
|
|
|
37694
37782
|
org_id_from_credentials: creds.orgId ?? null
|
|
37695
37783
|
}));
|
|
37696
37784
|
} catch (error2) {
|
|
37785
|
+
if (!checkTimings.has("auth")) addTiming(checkTimings, "auth", commandStartedMs);
|
|
37697
37786
|
checks.push(hold("auth", "auth_missing_or_expired", "CLI is not authenticated.", "foh auth login --web", {
|
|
37698
37787
|
message: error2 instanceof Error ? error2.message : String(error2)
|
|
37699
37788
|
}));
|
|
37700
37789
|
}
|
|
37701
37790
|
if (ctx.tokenPresent && !ctx.orgId) {
|
|
37702
37791
|
try {
|
|
37703
|
-
const orgs = await apiFetch("/v1/console/auth/my-orgs", { apiUrlOverride: opts.apiUrl });
|
|
37792
|
+
const orgs = await timedCheck(checkTimings, "org", () => apiFetch("/v1/console/auth/my-orgs", { apiUrlOverride: opts.apiUrl }));
|
|
37704
37793
|
const resolved = firstUsableOrgId(orgs);
|
|
37705
37794
|
if (resolved.orgId) {
|
|
37706
37795
|
ctx.orgId = resolved.orgId;
|
|
@@ -37728,10 +37817,10 @@ function registerProve(program3) {
|
|
|
37728
37817
|
checks.push(pass("agent_selection", "Using explicitly supplied agent.", { agent_id: ctx.agentId }));
|
|
37729
37818
|
} else {
|
|
37730
37819
|
try {
|
|
37731
|
-
const list = await apiFetch("/v1/console/agents", {
|
|
37820
|
+
const list = await timedCheck(checkTimings, "agent_selection", () => apiFetch("/v1/console/agents", {
|
|
37732
37821
|
orgId: ctx.orgId,
|
|
37733
37822
|
apiUrlOverride: opts.apiUrl
|
|
37734
|
-
});
|
|
37823
|
+
}));
|
|
37735
37824
|
const resolved = agentIdFromList(list);
|
|
37736
37825
|
if (resolved.agentId) {
|
|
37737
37826
|
ctx.agentId = resolved.agentId;
|
|
@@ -37754,11 +37843,11 @@ function registerProve(program3) {
|
|
|
37754
37843
|
}
|
|
37755
37844
|
if (ctx.agentId) {
|
|
37756
37845
|
try {
|
|
37757
|
-
const validation = await apiFetch(`/v1/console/agents/${ctx.agentId}/validate`, {
|
|
37846
|
+
const validation = await timedCheck(checkTimings, "agent_validation", () => apiFetch(`/v1/console/agents/${ctx.agentId}/validate`, {
|
|
37758
37847
|
method: "POST",
|
|
37759
37848
|
orgId: ctx.orgId,
|
|
37760
37849
|
apiUrlOverride: opts.apiUrl
|
|
37761
|
-
});
|
|
37850
|
+
}));
|
|
37762
37851
|
const issues = Array.isArray(validation.issues) ? validation.issues : [];
|
|
37763
37852
|
validationFingerprint = validation;
|
|
37764
37853
|
if (validation.ok === false || issues.length > 0) {
|
|
@@ -37771,10 +37860,10 @@ function registerProve(program3) {
|
|
|
37771
37860
|
}
|
|
37772
37861
|
if (ctx.orgId) {
|
|
37773
37862
|
try {
|
|
37774
|
-
const onboarding = await apiFetch(`/v1/console/org/${ctx.orgId}/onboarding`, {
|
|
37863
|
+
const onboarding = await timedCheck(checkTimings, "contact_channel", () => apiFetch(`/v1/console/org/${ctx.orgId}/onboarding`, {
|
|
37775
37864
|
orgId: ctx.orgId,
|
|
37776
37865
|
apiUrlOverride: opts.apiUrl
|
|
37777
|
-
});
|
|
37866
|
+
}));
|
|
37778
37867
|
const phoneNumber = typeof onboarding.phone_number === "string" && onboarding.phone_number.trim() ? onboarding.phone_number.trim() : null;
|
|
37779
37868
|
const provisioningStatus = typeof onboarding.provisioning_status === "string" ? onboarding.provisioning_status : null;
|
|
37780
37869
|
if (phoneNumber) {
|
|
@@ -37838,10 +37927,10 @@ function registerProve(program3) {
|
|
|
37838
37927
|
checks.push(skipped("voice_realtime_health", "operator_skipped", "Skipped by --skip-voice-health.", "foh voice realtime-health --json"));
|
|
37839
37928
|
} else {
|
|
37840
37929
|
try {
|
|
37841
|
-
const health = await apiFetch(
|
|
37930
|
+
const health = await timedCheck(checkTimings, "voice_realtime_health", () => apiFetch(
|
|
37842
37931
|
"/v1/console/realtime/health",
|
|
37843
37932
|
{ apiUrlOverride: opts.apiUrl }
|
|
37844
|
-
);
|
|
37933
|
+
));
|
|
37845
37934
|
const providers = Array.isArray(health.providers) ? health.providers : [];
|
|
37846
37935
|
if (providers.length === 0) {
|
|
37847
37936
|
checks.push(skipped("voice_realtime_health", "voice_health_no_providers", "Realtime voice health returned no providers.", "foh voice realtime-health --json"));
|
|
@@ -37859,13 +37948,14 @@ function registerProve(program3) {
|
|
|
37859
37948
|
}
|
|
37860
37949
|
}
|
|
37861
37950
|
try {
|
|
37862
|
-
const embed = await apiFetch("/v1/console/channels/widget/embed-snippet", {
|
|
37951
|
+
const embed = await timedCheck(checkTimings, "widget_embed", () => apiFetch("/v1/console/channels/widget/embed-snippet", {
|
|
37863
37952
|
orgId: ctx.orgId,
|
|
37864
37953
|
apiUrlOverride: opts.apiUrl,
|
|
37865
|
-
headers: { "x-agent-id": ctx.agentId }
|
|
37866
|
-
});
|
|
37954
|
+
headers: { "x-agent-id": String(ctx.agentId) }
|
|
37955
|
+
}));
|
|
37867
37956
|
const publicKey = publicKeyFromEmbedResponse(embed);
|
|
37868
37957
|
if (publicKey) {
|
|
37958
|
+
checkTimings.set("widget_channel", Math.max(checkTimings.get("widget_channel") ?? 0, checkTimings.get("widget_embed") ?? 0));
|
|
37869
37959
|
ctx.widgetPublicKey = publicKey;
|
|
37870
37960
|
checks.push(pass("widget_channel", "Widget channel is available in read-only proof mode.", {
|
|
37871
37961
|
public_key_present: true,
|
|
@@ -37880,12 +37970,12 @@ function registerProve(program3) {
|
|
|
37880
37970
|
} catch (error2) {
|
|
37881
37971
|
if (mutationMode === "ensure") {
|
|
37882
37972
|
try {
|
|
37883
|
-
const ensure = await apiFetch("/v1/console/channels/widget/ensure", {
|
|
37973
|
+
const ensure = await timedCheck(checkTimings, "widget_channel", () => apiFetch("/v1/console/channels/widget/ensure", {
|
|
37884
37974
|
method: "POST",
|
|
37885
37975
|
body: JSON.stringify({ agentId: ctx.agentId }),
|
|
37886
37976
|
orgId: ctx.orgId,
|
|
37887
37977
|
apiUrlOverride: opts.apiUrl
|
|
37888
|
-
});
|
|
37978
|
+
}));
|
|
37889
37979
|
const publicKey = publicKeyFromEnsureResponse(ensure);
|
|
37890
37980
|
if (!publicKey) {
|
|
37891
37981
|
checks.push(hold("widget_channel", "widget_public_key_missing", "Widget channel ensure returned no public key.", `foh widget ensure --agent ${ctx.agentId} --json`, ensure));
|
|
@@ -37915,7 +38005,7 @@ function registerProve(program3) {
|
|
|
37915
38005
|
checks.push(skipped("widget_smoke", "widget_public_key_required", "Skipped because widget public key is unavailable.", `foh widget ensure --agent ${ctx.agentId} --json`));
|
|
37916
38006
|
} else {
|
|
37917
38007
|
try {
|
|
37918
|
-
const smoke = await runWidgetSmoke(ctx.widgetPublicKey, opts.apiUrl);
|
|
38008
|
+
const smoke = await timedCheck(checkTimings, "widget_smoke", () => runWidgetSmoke(ctx.widgetPublicKey, opts.apiUrl));
|
|
37919
38009
|
ctx.conversationId = smoke.conversation_id;
|
|
37920
38010
|
ctx.traceIds = smoke.trace_ids;
|
|
37921
38011
|
ctx.correlationIds = smoke.correlation_ids;
|
|
@@ -37943,7 +38033,7 @@ function registerProve(program3) {
|
|
|
37943
38033
|
const agentId = ctx.agentId;
|
|
37944
38034
|
const adaptiveRuns = Math.max(1, Number(opts.certAdaptiveRuns ?? 30) || 30);
|
|
37945
38035
|
const maxImprovementRounds = Math.max(0, Math.min(5, Number(opts.certMaxImprovementRounds ?? 1) || 1));
|
|
37946
|
-
const cached2 = await withProofCache({
|
|
38036
|
+
const cached2 = await timedCheck(checkTimings, "simulation_certification", () => withProofCache({
|
|
37947
38037
|
cacheDir: opts.proofCacheDir,
|
|
37948
38038
|
kind: "simulation_certification",
|
|
37949
38039
|
keyParts: {
|
|
@@ -37961,7 +38051,7 @@ function registerProve(program3) {
|
|
|
37961
38051
|
maxImprovementRounds,
|
|
37962
38052
|
orgId: ctx.orgId,
|
|
37963
38053
|
apiUrlOverride: opts.apiUrl
|
|
37964
|
-
}));
|
|
38054
|
+
})));
|
|
37965
38055
|
const loop = cached2.value;
|
|
37966
38056
|
const loopWithCache = {
|
|
37967
38057
|
...loop,
|
|
@@ -37975,6 +38065,7 @@ function registerProve(program3) {
|
|
|
37975
38065
|
attempts: loop.attempts?.length ?? 0,
|
|
37976
38066
|
improvement_runs: loop.improvement_runs,
|
|
37977
38067
|
scenario_summary: loop.certificate?.scenario_summary,
|
|
38068
|
+
performance_summary: loop.certificate?.performance_summary ?? null,
|
|
37978
38069
|
proof_cache: cached2.metadata
|
|
37979
38070
|
}));
|
|
37980
38071
|
}
|
|
@@ -37991,8 +38082,9 @@ function registerProve(program3) {
|
|
|
37991
38082
|
checks.push(skipped("widget_smoke", "agent_required", "Skipped until an agent is selected.", "foh agent list --json"));
|
|
37992
38083
|
checks.push(skipped("simulation_certification", "agent_required", "Skipped until an agent is selected.", "foh agent list --json"));
|
|
37993
38084
|
}
|
|
37994
|
-
const
|
|
37995
|
-
const
|
|
38085
|
+
const timedChecks = enrichCheckTimings(checks, checkTimings);
|
|
38086
|
+
const status = hasBlockingChecks(timedChecks) ? "hold" : "pass";
|
|
38087
|
+
const nextCommands = Array.from(new Set(timedChecks.map((check2) => check2.next_command).filter((command) => Boolean(command))));
|
|
37996
38088
|
if (status === "pass" && ctx.agentId) {
|
|
37997
38089
|
nextCommands.push(`foh agent publish --agent ${ctx.agentId} --json`);
|
|
37998
38090
|
}
|
|
@@ -38012,10 +38104,11 @@ function registerProve(program3) {
|
|
|
38012
38104
|
trace_ids: ctx.traceIds,
|
|
38013
38105
|
correlation_ids: ctx.correlationIds
|
|
38014
38106
|
},
|
|
38015
|
-
checks,
|
|
38107
|
+
checks: timedChecks,
|
|
38016
38108
|
nextCommands,
|
|
38017
38109
|
extra: {
|
|
38018
|
-
generated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
38110
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
38111
|
+
timing: proofTimingSummary(timedChecks, commandStartedMs)
|
|
38019
38112
|
}
|
|
38020
38113
|
}));
|
|
38021
38114
|
const artifactPath = opts.out ? writeSignedJsonArtifact(String(opts.out), report) : void 0;
|
|
@@ -39805,15 +39898,82 @@ function collapseCommandRecords(records) {
|
|
|
39805
39898
|
}
|
|
39806
39899
|
return order.map((id) => byId.get(id)).filter((record2) => Boolean(record2));
|
|
39807
39900
|
}
|
|
39901
|
+
function readCommandOutputJson(runDir, command) {
|
|
39902
|
+
const artifact = typeof command.output_artifact === "string" && command.output_artifact.trim() ? command.output_artifact.trim() : "";
|
|
39903
|
+
if (!artifact || artifact.includes("..") || artifact.includes("/") || artifact.includes("\\")) return null;
|
|
39904
|
+
const artifactPath = (0, import_path15.join)(runDir, artifact);
|
|
39905
|
+
if (!(0, import_fs16.existsSync)(artifactPath)) return null;
|
|
39906
|
+
try {
|
|
39907
|
+
const text = (0, import_fs16.readFileSync)(artifactPath, "utf8");
|
|
39908
|
+
const firstObject = text.indexOf("{");
|
|
39909
|
+
const lastObject = text.lastIndexOf("}");
|
|
39910
|
+
if (firstObject < 0 || lastObject <= firstObject) return null;
|
|
39911
|
+
return JSON.parse(text.slice(firstObject, lastObject + 1));
|
|
39912
|
+
} catch {
|
|
39913
|
+
return null;
|
|
39914
|
+
}
|
|
39915
|
+
}
|
|
39916
|
+
function commandTimingBreakdown(command, output) {
|
|
39917
|
+
const schemaVersion = String(output?.schema_version || "");
|
|
39918
|
+
if (schemaVersion === "foh_cli_proof_report.v1") {
|
|
39919
|
+
const timing = asObject(output?.timing) || {};
|
|
39920
|
+
const cacheSources = /* @__PURE__ */ new Map();
|
|
39921
|
+
let cacheWaitMs = 0;
|
|
39922
|
+
let cacheHitCount = 0;
|
|
39923
|
+
let cacheMissCount = 0;
|
|
39924
|
+
for (const check2 of toArray2(output?.checks)) {
|
|
39925
|
+
const detail = asObject(check2)?.detail;
|
|
39926
|
+
const proofCache = asObject(asObject(detail)?.proof_cache);
|
|
39927
|
+
if (!proofCache) continue;
|
|
39928
|
+
if (proofCache["hit"] === true) cacheHitCount += 1;
|
|
39929
|
+
if (proofCache["miss"] === true) cacheMissCount += 1;
|
|
39930
|
+
cacheWaitMs += Number(proofCache["waited_ms"] || 0);
|
|
39931
|
+
increment(cacheSources, proofCache["source"] || "unknown");
|
|
39932
|
+
}
|
|
39933
|
+
return {
|
|
39934
|
+
kind: "proof",
|
|
39935
|
+
command: command.command || "",
|
|
39936
|
+
total_ms: Number(timing.total_ms || command.duration_ms || 0),
|
|
39937
|
+
total_check_duration_ms: Number(timing.total_check_duration_ms || 0),
|
|
39938
|
+
slow_checks: toArray2(timing.slow_checks).slice(0, 5),
|
|
39939
|
+
cache_wait_ms: cacheWaitMs,
|
|
39940
|
+
cache_hit_count: cacheHitCount,
|
|
39941
|
+
cache_miss_count: cacheMissCount,
|
|
39942
|
+
cache_sources: ranked(cacheSources)
|
|
39943
|
+
};
|
|
39944
|
+
}
|
|
39945
|
+
if (schemaVersion === "foh_certification_run.v1") {
|
|
39946
|
+
const timing = asObject(output?.timing) || {};
|
|
39947
|
+
const certificate = asObject(output?.certificate) || {};
|
|
39948
|
+
return {
|
|
39949
|
+
kind: "certification",
|
|
39950
|
+
command: command.command || "",
|
|
39951
|
+
total_ms: Number(timing.total_ms || command.duration_ms || 0),
|
|
39952
|
+
api_ms: Number(timing.api_ms || 0),
|
|
39953
|
+
status: output?.status || null,
|
|
39954
|
+
reason_code: output?.reason_code || null,
|
|
39955
|
+
scenario_summary: certificate.scenario_summary || null
|
|
39956
|
+
};
|
|
39957
|
+
}
|
|
39958
|
+
return null;
|
|
39959
|
+
}
|
|
39808
39960
|
function analyzeRunArtifacts(runPath, run, cwd) {
|
|
39809
39961
|
const runDir = (0, import_path15.dirname)(runPath);
|
|
39810
39962
|
const commands = collapseCommandRecords(readNdjson((0, import_path15.join)(runDir, "commands.ndjson")));
|
|
39811
39963
|
const reasonCounts = /* @__PURE__ */ new Map();
|
|
39812
39964
|
const slowSteps = [];
|
|
39965
|
+
const timingBreakdowns = [];
|
|
39813
39966
|
let completed = 0;
|
|
39814
39967
|
let withDuration = 0;
|
|
39815
39968
|
let totalDuration = 0;
|
|
39816
39969
|
for (const command of commands) {
|
|
39970
|
+
const output = readCommandOutputJson(runDir, command);
|
|
39971
|
+
const breakdown = commandTimingBreakdown(command, output);
|
|
39972
|
+
if (breakdown) timingBreakdowns.push({
|
|
39973
|
+
run_id: run.run_id,
|
|
39974
|
+
run_path: (0, import_path15.relative)(cwd, runPath).replaceAll("\\", "/"),
|
|
39975
|
+
...breakdown
|
|
39976
|
+
});
|
|
39817
39977
|
if (command.phase === "completed" || command.completed_at) completed += 1;
|
|
39818
39978
|
if (typeof command.duration_ms === "number") {
|
|
39819
39979
|
withDuration += 1;
|
|
@@ -39858,6 +40018,7 @@ function analyzeRunArtifacts(runPath, run, cwd) {
|
|
|
39858
40018
|
total_command_duration_ms: totalDuration,
|
|
39859
40019
|
command_reason_codes: ranked(reasonCounts),
|
|
39860
40020
|
slow_steps: slowSteps.sort((a, b) => Number(b.duration_ms) - Number(a.duration_ms)).slice(0, 10),
|
|
40021
|
+
timing_breakdowns: timingBreakdowns,
|
|
39861
40022
|
docs_pages_observed: Array.from(docs).sort(),
|
|
39862
40023
|
codex_command_execution_completed_count: codexCommandExecutions,
|
|
39863
40024
|
codex_failed_exit_code_count: codexFailedExitCodes
|
|
@@ -39875,6 +40036,7 @@ function summarizeExternalAgentRuns(options) {
|
|
|
39875
40036
|
const commandReasonCounts = /* @__PURE__ */ new Map();
|
|
39876
40037
|
const docsCounts = /* @__PURE__ */ new Map();
|
|
39877
40038
|
const slowSteps = [];
|
|
40039
|
+
const timingBreakdowns = [];
|
|
39878
40040
|
let manualInterventions = 0;
|
|
39879
40041
|
let commandCount = 0;
|
|
39880
40042
|
let completedCommandCount = 0;
|
|
@@ -39900,6 +40062,7 @@ function summarizeExternalAgentRuns(options) {
|
|
|
39900
40062
|
codexCommandExecutions += Number(artifactSummary.codex_command_execution_completed_count || 0);
|
|
39901
40063
|
codexFailedExitCodes += Number(artifactSummary.codex_failed_exit_code_count || 0);
|
|
39902
40064
|
for (const row of toArray2(artifactSummary.slow_steps)) slowSteps.push(row);
|
|
40065
|
+
for (const row of toArray2(artifactSummary.timing_breakdowns)) timingBreakdowns.push(row);
|
|
39903
40066
|
for (const row of toArray2(artifactSummary.command_reason_codes)) {
|
|
39904
40067
|
const entry = asObject(row);
|
|
39905
40068
|
if (entry) increment(commandReasonCounts, entry.key, Number(entry.count || 1));
|
|
@@ -39936,7 +40099,8 @@ function summarizeExternalAgentRuns(options) {
|
|
|
39936
40099
|
commands_with_duration_count: commandsWithDurationCount,
|
|
39937
40100
|
total_command_duration_ms: totalCommandDurationMs,
|
|
39938
40101
|
command_reason_codes: commandReasonCodes2,
|
|
39939
|
-
slow_steps: slowSteps.sort((a, b) => Number(b.duration_ms || 0) - Number(a.duration_ms || 0) || String(a.command || "").localeCompare(String(b.command || ""))).slice(0, 20)
|
|
40102
|
+
slow_steps: slowSteps.sort((a, b) => Number(b.duration_ms || 0) - Number(a.duration_ms || 0) || String(a.command || "").localeCompare(String(b.command || ""))).slice(0, 20),
|
|
40103
|
+
timing_breakdowns: timingBreakdowns.sort((a, b) => Number(b.total_ms || 0) - Number(a.total_ms || 0) || String(a.command || "").localeCompare(String(b.command || ""))).slice(0, 20)
|
|
39940
40104
|
},
|
|
39941
40105
|
codex_telemetry: {
|
|
39942
40106
|
command_execution_completed_count: codexCommandExecutions,
|