gh-glance 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -15
- package/README.md +31 -20
- package/index.mjs +1220 -241
- package/package.json +1 -1
package/index.mjs
CHANGED
|
@@ -162,6 +162,11 @@ const AUTH_RETRY_MS = [30_000];
|
|
|
162
162
|
// byte-identical-idle-frame property the rest of this file works to keep.
|
|
163
163
|
const STALE_AFTER_MS = 30_000;
|
|
164
164
|
|
|
165
|
+
// A coordination hold must persist long enough to be meaningful before it
|
|
166
|
+
// takes the notice row. Short startup and lock-contention states recover on
|
|
167
|
+
// their own and should not present themselves as user-visible failures.
|
|
168
|
+
const COORDINATION_NOTICE_AFTER_MS = 2_000;
|
|
169
|
+
|
|
165
170
|
// Mutable because the fetchers below are defined before argv is parsed, and the
|
|
166
171
|
// argv block is what fills this in. Everything here has a working default, so
|
|
167
172
|
// the zero-argument invocation the README documents behaves exactly as before.
|
|
@@ -390,6 +395,24 @@ function isMissingRemote(err) {
|
|
|
390
395
|
return /(?:failed to determine base repo:\s*)?no git remotes found/i.test(errText(err));
|
|
391
396
|
}
|
|
392
397
|
|
|
398
|
+
const UNUSABLE_OUTPUT_CODE = "GH_GLANCE_UNUSABLE_OUTPUT";
|
|
399
|
+
|
|
400
|
+
// Only JSON parsed from a successful gh stdout stream goes through this seam.
|
|
401
|
+
// Marking the SyntaxError here distinguishes our own failed parse from a gh
|
|
402
|
+
// stderr message that happens to use the same words.
|
|
403
|
+
function parseJsonOutput(raw) {
|
|
404
|
+
try {
|
|
405
|
+
return JSON.parse(raw);
|
|
406
|
+
} catch (error) {
|
|
407
|
+
if (error instanceof SyntaxError) error.code = UNUSABLE_OUTPUT_CODE;
|
|
408
|
+
throw error;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function isUnusableOutput(err) {
|
|
413
|
+
return err?.code === UNUSABLE_OUTPUT_CODE;
|
|
414
|
+
}
|
|
415
|
+
|
|
393
416
|
// The one place the three predicates are turned into a verdict. Both consumers
|
|
394
417
|
// go through it -- the fetcher, to choose a note and a backoff ladder, and
|
|
395
418
|
// `--doctor`, to report what gh-glance concluded -- so the report cannot claim
|
|
@@ -409,6 +432,7 @@ function classify(err) {
|
|
|
409
432
|
if (isRateLimited(err)) return "rate-limited";
|
|
410
433
|
if (isAuthProblem(err)) return "auth-problem";
|
|
411
434
|
if (isUnavailable(err)) return "unavailable";
|
|
435
|
+
if (isUnusableOutput(err)) return "unusable-output";
|
|
412
436
|
return "other";
|
|
413
437
|
}
|
|
414
438
|
|
|
@@ -624,6 +648,92 @@ async function runGh(args, { signal, operation } = {}) {
|
|
|
624
648
|
}
|
|
625
649
|
}
|
|
626
650
|
|
|
651
|
+
function parseGhApiResponse(stdout) {
|
|
652
|
+
const text = typeof stdout === "string" ? stdout : String(stdout ?? "");
|
|
653
|
+
const separator = /\r?\n\r?\n/.exec(text);
|
|
654
|
+
const prefix = separator ? text.slice(0, separator.index) : text;
|
|
655
|
+
const body = separator ? text.slice(separator.index + separator[0].length) : "";
|
|
656
|
+
const lines = prefix.split(/\r?\n/);
|
|
657
|
+
const statusMatch = /^HTTP\/\d(?:\.\d+)?\s+(\d{3})(?:\s|$)/i.exec(lines[0] ?? "");
|
|
658
|
+
const headers = {};
|
|
659
|
+
for (const line of lines.slice(1)) {
|
|
660
|
+
const colon = line.indexOf(":");
|
|
661
|
+
if (colon <= 0) continue;
|
|
662
|
+
const name = line.slice(0, colon).trim().toLowerCase();
|
|
663
|
+
if (name) headers[name] = line.slice(colon + 1).trim();
|
|
664
|
+
}
|
|
665
|
+
return {
|
|
666
|
+
status: statusMatch ? Number(statusMatch[1]) : null,
|
|
667
|
+
headers,
|
|
668
|
+
body,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function pickRateLimit(headers) {
|
|
673
|
+
if (!headers || typeof headers !== "object" || Array.isArray(headers)) return null;
|
|
674
|
+
const integer = (name) => {
|
|
675
|
+
const value = Number(headers[name]);
|
|
676
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
677
|
+
};
|
|
678
|
+
const resource = typeof headers["x-ratelimit-resource"] === "string"
|
|
679
|
+
? headers["x-ratelimit-resource"].trim().toLowerCase()
|
|
680
|
+
: "";
|
|
681
|
+
const limit = integer("x-ratelimit-limit");
|
|
682
|
+
const used = integer("x-ratelimit-used");
|
|
683
|
+
const remaining = integer("x-ratelimit-remaining");
|
|
684
|
+
const reset = integer("x-ratelimit-reset");
|
|
685
|
+
if (!RATE_RESOURCES.includes(resource) || [limit, used, remaining, reset].includes(null)) return null;
|
|
686
|
+
return { resource, limit, used, remaining, resetMs: reset * 1000 };
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
function responseHeaderObservation(rateLimit, cost, receivedAt = Date.now()) {
|
|
690
|
+
return rateLimit ? {
|
|
691
|
+
...rateLimit,
|
|
692
|
+
source: "response-header",
|
|
693
|
+
receivedAt,
|
|
694
|
+
cost,
|
|
695
|
+
} : null;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function ghApiArgs(args, etag = null) {
|
|
699
|
+
return [
|
|
700
|
+
"api",
|
|
701
|
+
"-i",
|
|
702
|
+
...args,
|
|
703
|
+
...(typeof etag === "string" && etag.length > 0 ? ["-H", `If-None-Match: ${etag}`] : []),
|
|
704
|
+
];
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
async function ghApi(args, { operation, signal, etag = null, run = runGh } = {}) {
|
|
708
|
+
const argv = ghApiArgs(args, etag);
|
|
709
|
+
try {
|
|
710
|
+
const parsed = parseGhApiResponse(await run(argv, { operation, signal }));
|
|
711
|
+
return {
|
|
712
|
+
status: parsed.status,
|
|
713
|
+
etag: parsed.headers.etag ?? null,
|
|
714
|
+
rateLimit: pickRateLimit(parsed.headers),
|
|
715
|
+
body: parsed.body,
|
|
716
|
+
};
|
|
717
|
+
} catch (error) {
|
|
718
|
+
const parsed = parseGhApiResponse(error?.stdout);
|
|
719
|
+
if (parsed.status === 304) {
|
|
720
|
+
return {
|
|
721
|
+
status: 304,
|
|
722
|
+
etag: parsed.headers.etag ?? null,
|
|
723
|
+
rateLimit: pickRateLimit(parsed.headers),
|
|
724
|
+
body: "",
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
if (parsed.status !== null) {
|
|
728
|
+
error.apiResponse = parsed;
|
|
729
|
+
const rateLimit = pickRateLimit(parsed.headers);
|
|
730
|
+
const observation = responseHeaderObservation(rateLimit, 0);
|
|
731
|
+
if (observation) error.budgetObservations = [observation];
|
|
732
|
+
}
|
|
733
|
+
throw error;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
627
737
|
// The target as `gh` spells it: host-qualified when a host was given, the bare
|
|
628
738
|
// slug otherwise -- which is exactly the [HOST/]OWNER/REPO form `gh --repo`
|
|
629
739
|
// documents.
|
|
@@ -872,40 +982,161 @@ function createFailureContextCoordinator({ resolve, commit, fallback }) {
|
|
|
872
982
|
return { ensure, invalidate };
|
|
873
983
|
}
|
|
874
984
|
|
|
875
|
-
function
|
|
985
|
+
function actionsRunsArgs(limit) {
|
|
876
986
|
return [
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
"
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
987
|
+
apiPath(`repos/{owner}/{repo}/actions/runs?exclude_pull_requests=true&per_page=${limit}`),
|
|
988
|
+
...apiHostArgs(),
|
|
989
|
+
"--jq",
|
|
990
|
+
"[.workflow_runs[] | {databaseId: .id, displayTitle: .display_title, number: .run_number, headBranch: .head_branch, status, conclusion, startedAt: .run_started_at, updatedAt: .updated_at, workflowId: .workflow_id}]",
|
|
991
|
+
];
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
function actionsWorkflowsArgs() {
|
|
995
|
+
return [
|
|
996
|
+
apiPath("repos/{owner}/{repo}/actions/workflows?page=1&per_page=100"),
|
|
997
|
+
...apiHostArgs(),
|
|
998
|
+
"--jq",
|
|
999
|
+
"[.workflows[] | {id, name}]",
|
|
884
1000
|
];
|
|
885
1001
|
}
|
|
886
1002
|
|
|
887
|
-
|
|
888
|
-
const
|
|
1003
|
+
function parseActionsBodies(runsBody, workflowsBody) {
|
|
1004
|
+
const workflows = new Map(parseJsonOutput(workflowsBody).map((workflow) => [
|
|
1005
|
+
workflow.id,
|
|
1006
|
+
safe(workflow.name),
|
|
1007
|
+
]));
|
|
1008
|
+
return parseJsonOutput(runsBody).map((run) => ({
|
|
1009
|
+
databaseId: run.databaseId,
|
|
1010
|
+
displayTitle: safe(run.displayTitle),
|
|
1011
|
+
workflowName: workflows.get(run.workflowId) ?? "",
|
|
1012
|
+
number: run.number,
|
|
1013
|
+
headBranch: safe(run.headBranch),
|
|
1014
|
+
status: run.status,
|
|
1015
|
+
conclusion: run.conclusion,
|
|
1016
|
+
startedAt: run.startedAt,
|
|
1017
|
+
updatedAt: run.updatedAt,
|
|
1018
|
+
}));
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
function entityKey(tab, path) {
|
|
1022
|
+
return `${tab}\0${path}`;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
function cachedEntity(entities, tab, path) {
|
|
1026
|
+
const value = entities instanceof Map ? entities.get(entityKey(tab, path)) : null;
|
|
1027
|
+
return typeof value?.etag === "string" && typeof value?.body === "string" ? value : null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
async function fetchConditionalEntity({
|
|
1031
|
+
tab,
|
|
1032
|
+
args,
|
|
1033
|
+
operation,
|
|
1034
|
+
signal,
|
|
1035
|
+
force,
|
|
1036
|
+
entities,
|
|
1037
|
+
request = ghApi,
|
|
1038
|
+
}) {
|
|
1039
|
+
const path = args[0];
|
|
1040
|
+
const cached = cachedEntity(entities, tab, path);
|
|
1041
|
+
const response = await request(args, {
|
|
1042
|
+
operation,
|
|
1043
|
+
signal,
|
|
1044
|
+
etag: force ? null : cached?.etag ?? null,
|
|
1045
|
+
});
|
|
1046
|
+
const observation = responseHeaderObservation(
|
|
1047
|
+
response.rateLimit,
|
|
1048
|
+
response.status === 200 ? 1 : 0,
|
|
1049
|
+
);
|
|
1050
|
+
if (response.status === 304) {
|
|
1051
|
+
return {
|
|
1052
|
+
path,
|
|
1053
|
+
status: 304,
|
|
1054
|
+
body: cached?.body ?? null,
|
|
1055
|
+
staged: null,
|
|
1056
|
+
observations: observation ? [observation] : [],
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
889
1059
|
return {
|
|
890
|
-
|
|
1060
|
+
path,
|
|
1061
|
+
status: response.status,
|
|
1062
|
+
body: response.body,
|
|
1063
|
+
staged: response.status === 200
|
|
1064
|
+
? [entityKey(tab, path), typeof response.etag === "string"
|
|
1065
|
+
? { etag: response.etag, body: response.body }
|
|
1066
|
+
: null]
|
|
1067
|
+
: null,
|
|
1068
|
+
observations: observation ? [observation] : [],
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function conditionalBatchResult(responses, previousRaw, joinedRaw = null) {
|
|
1073
|
+
const list = Array.isArray(responses) ? responses : [];
|
|
1074
|
+
let allNotModified = list.length > 0;
|
|
1075
|
+
let restSpent = 0;
|
|
1076
|
+
let stagedEntities = null;
|
|
1077
|
+
const observations = [];
|
|
1078
|
+
for (const response of list) {
|
|
1079
|
+
if (response?.status !== 304 || typeof response.body !== "string") allNotModified = false;
|
|
1080
|
+
if (response?.status === 200) restSpent += 1;
|
|
1081
|
+
if (Array.isArray(response?.observations)) observations.push(...response.observations);
|
|
1082
|
+
if (!response?.staged) continue;
|
|
1083
|
+
stagedEntities ??= new Map();
|
|
1084
|
+
stagedEntities.set(...response.staged);
|
|
1085
|
+
}
|
|
1086
|
+
return {
|
|
1087
|
+
raw: allNotModified
|
|
1088
|
+
? previousRaw
|
|
1089
|
+
: joinedRaw ?? list.map((response) => response?.body ?? "").join("\0"),
|
|
1090
|
+
allNotModified,
|
|
1091
|
+
restSpent,
|
|
1092
|
+
stagedEntities,
|
|
1093
|
+
observations,
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function publishStagedEntities(entities, staged, transitionKind) {
|
|
1098
|
+
if (!(entities instanceof Map) || !(staged instanceof Map) ||
|
|
1099
|
+
!["changed", "unchanged"].includes(transitionKind)) return false;
|
|
1100
|
+
for (const [key, value] of staged) {
|
|
1101
|
+
if (value === null) entities.delete(key);
|
|
1102
|
+
else entities.set(key, value);
|
|
1103
|
+
}
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
async function fetchActions(limit, signal, {
|
|
1108
|
+
entities = new Map(),
|
|
1109
|
+
force = false,
|
|
1110
|
+
previousRaw = null,
|
|
1111
|
+
} = {}) {
|
|
1112
|
+
const responses = await Promise.all([
|
|
1113
|
+
fetchConditionalEntity({
|
|
1114
|
+
tab: "actions",
|
|
1115
|
+
args: actionsRunsArgs(limit),
|
|
1116
|
+
operation: "tab:actions-runs",
|
|
1117
|
+
signal,
|
|
1118
|
+
force,
|
|
1119
|
+
entities,
|
|
1120
|
+
}),
|
|
1121
|
+
fetchConditionalEntity({
|
|
1122
|
+
tab: "actions",
|
|
1123
|
+
args: actionsWorkflowsArgs(),
|
|
1124
|
+
operation: "tab:actions-workflows",
|
|
1125
|
+
signal,
|
|
1126
|
+
force,
|
|
1127
|
+
entities,
|
|
1128
|
+
}),
|
|
1129
|
+
]);
|
|
1130
|
+
const [runs, workflows] = responses;
|
|
1131
|
+
const batch = conditionalBatchResult(responses, previousRaw);
|
|
1132
|
+
return {
|
|
1133
|
+
raw: batch.raw,
|
|
891
1134
|
limit,
|
|
892
|
-
|
|
893
|
-
// re-derived from the argv shape -- a second copy of the cost model is the
|
|
894
|
-
// drift this file warns about repeatedly.
|
|
895
|
-
restSpent: REST_PER_FETCH.actions,
|
|
1135
|
+
restSpent: batch.restSpent,
|
|
896
1136
|
graphqlSpent: GRAPHQL_PER_FETCH.actions,
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
displayTitle: safe(r.displayTitle),
|
|
901
|
-
workflowName: safe(r.workflowName),
|
|
902
|
-
number: r.number,
|
|
903
|
-
headBranch: safe(r.headBranch),
|
|
904
|
-
status: r.status,
|
|
905
|
-
conclusion: r.conclusion,
|
|
906
|
-
startedAt: r.startedAt,
|
|
907
|
-
updatedAt: r.updatedAt,
|
|
908
|
-
})),
|
|
1137
|
+
stagedEntities: batch.stagedEntities,
|
|
1138
|
+
observations: batch.observations,
|
|
1139
|
+
parse: () => parseActionsBodies(runs.body ?? "", workflows.body ?? ""),
|
|
909
1140
|
};
|
|
910
1141
|
}
|
|
911
1142
|
|
|
@@ -944,7 +1175,7 @@ async function fetchIssues(signal) {
|
|
|
944
1175
|
restSpent: REST_PER_FETCH.issues,
|
|
945
1176
|
graphqlSpent: GRAPHQL_PER_FETCH.issues,
|
|
946
1177
|
parse: () =>
|
|
947
|
-
|
|
1178
|
+
parseJsonOutput(raw).map((i) => ({
|
|
948
1179
|
number: i.number,
|
|
949
1180
|
title: safe(i.title),
|
|
950
1181
|
author: safe(i.author?.login ?? ""),
|
|
@@ -978,7 +1209,7 @@ async function fetchPRs(signal) {
|
|
|
978
1209
|
restSpent: REST_PER_FETCH.prs,
|
|
979
1210
|
graphqlSpent: GRAPHQL_PER_FETCH.prs,
|
|
980
1211
|
parse: () =>
|
|
981
|
-
|
|
1212
|
+
parseJsonOutput(raw).map((p) => ({
|
|
982
1213
|
number: p.number,
|
|
983
1214
|
title: safe(p.title),
|
|
984
1215
|
author: safe(p.author?.login ?? ""),
|
|
@@ -1086,6 +1317,8 @@ function tabRequestCost(tab) {
|
|
|
1086
1317
|
// individual calls covered by the tab's six-call upfront reservation.
|
|
1087
1318
|
const OPERATION_COSTS = Object.freeze({
|
|
1088
1319
|
...Object.fromEntries(TAB_KEYS.map((tab) => [`tab:${tab}`, tabRequestCost(tab)])),
|
|
1320
|
+
"tab:actions-runs": { core: 1, graphql: 0 },
|
|
1321
|
+
"tab:actions-workflows": { core: 1, graphql: 0 },
|
|
1089
1322
|
"tab:security-endpoint": { core: 1, graphql: 0 },
|
|
1090
1323
|
"failure-context:repository": { core: 0, graphql: 1 },
|
|
1091
1324
|
"failure-context:auth": { core: 0, graphql: 0 },
|
|
@@ -1093,10 +1326,15 @@ const OPERATION_COSTS = Object.freeze({
|
|
|
1093
1326
|
"open:issues": { core: 0, graphql: GRAPHQL_PER_FETCH.issues },
|
|
1094
1327
|
"open:prs": { core: 0, graphql: GRAPHQL_PER_FETCH.prs },
|
|
1095
1328
|
"doctor:repository": { core: 0, graphql: 1 },
|
|
1096
|
-
"doctor:actions":
|
|
1329
|
+
"doctor:actions-runs": { core: 1, graphql: 0 },
|
|
1330
|
+
"doctor:actions-workflows": { core: 1, graphql: 0 },
|
|
1097
1331
|
"doctor:issues": tabRequestCost("issues"),
|
|
1098
1332
|
"doctor:prs": tabRequestCost("prs"),
|
|
1099
1333
|
"doctor:security-endpoint": { core: 1, graphql: 0 },
|
|
1334
|
+
// The claimed observer is the one control-plane exception to normal data
|
|
1335
|
+
// admission, but its worst-case vector is still one core unit: the first 200
|
|
1336
|
+
// can spend it before an ETag makes later observations free.
|
|
1337
|
+
"budget-core-observer": { core: 1, graphql: 0 },
|
|
1100
1338
|
"rate-limit": { core: 0, graphql: 0 },
|
|
1101
1339
|
"version": { core: 0, graphql: 0 },
|
|
1102
1340
|
"auth-status": { core: 0, graphql: 0 },
|
|
@@ -1111,7 +1349,15 @@ function operationCost(operation) {
|
|
|
1111
1349
|
// is for the user's own `gh` and `git` commands.
|
|
1112
1350
|
const BUDGET_SAFETY = 0.8;
|
|
1113
1351
|
const BUDGET_RESERVE_FRACTION = 1 - BUDGET_SAFETY;
|
|
1352
|
+
// How often to re-read the budget. `gh api rate_limit` does not count against
|
|
1353
|
+
// the limit (verified, delta 0 -- see rateBudget) but it is still a subprocess,
|
|
1354
|
+
// so once a minute rather than once a tick: the quantity it measures moves on
|
|
1355
|
+
// the scale of minutes.
|
|
1356
|
+
const BUDGET_PROBE_MS = 60_000;
|
|
1114
1357
|
const BUDGET_SNAPSHOT_TTL_MS = 65_000;
|
|
1358
|
+
// GraphQL has no response-header owner. One unusable minute sample must not
|
|
1359
|
+
// close admission, but two consecutive misses still do: 2 probes + 5s grace.
|
|
1360
|
+
const GRAPHQL_BUDGET_SNAPSHOT_TTL_MS = 2 * BUDGET_PROBE_MS + 5_000;
|
|
1115
1361
|
const GOVERNOR_HEARTBEAT_MS = 20_000;
|
|
1116
1362
|
const GOVERNOR_LEASE_TTL_MS = 90_000;
|
|
1117
1363
|
const GOVERNOR_PROBE_LEASE_MS = 70_000;
|
|
@@ -1120,11 +1366,27 @@ const BUDGET_RESET_GRACE_MS = 2_000;
|
|
|
1120
1366
|
const GOVERNOR_PHASE_WINDOW_MS = 5_000;
|
|
1121
1367
|
const BUDGET_WINDOW_MS = 3_600_000;
|
|
1122
1368
|
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1369
|
+
function budgetSnapshotTtl(resource) {
|
|
1370
|
+
return resource === "graphql" ? GRAPHQL_BUDGET_SNAPSHOT_TTL_MS : BUDGET_SNAPSHOT_TTL_MS;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
function sharedLaneProvenance(value) {
|
|
1374
|
+
return value?.waitCause === "shared-lane" &&
|
|
1375
|
+
Number.isSafeInteger(value?.sharingCount) && value.sharingCount > 1
|
|
1376
|
+
? { waitCause: "shared-lane", sharingCount: value.sharingCount }
|
|
1377
|
+
: {};
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
function sharedLaneEvidence(value) {
|
|
1381
|
+
const provenance = sharedLaneProvenance(value);
|
|
1382
|
+
const ownerLeaseIds = Array.isArray(value?.sharingOwnerLeaseIds)
|
|
1383
|
+
? [...new Set(value.sharingOwnerLeaseIds.filter((leaseId) =>
|
|
1384
|
+
typeof leaseId === "string" && leaseId.length > 0))]
|
|
1385
|
+
: [];
|
|
1386
|
+
return provenance.waitCause && ownerLeaseIds.length > 0
|
|
1387
|
+
? { ...provenance, sharingOwnerLeaseIds: ownerLeaseIds }
|
|
1388
|
+
: {};
|
|
1389
|
+
}
|
|
1128
1390
|
|
|
1129
1391
|
// Below this many completed shared calls in a probe window, external-spend
|
|
1130
1392
|
// inference is noise. Under the threshold the loop retains the previous factor.
|
|
@@ -1175,6 +1437,23 @@ function leaseFor(leases, leaseId) {
|
|
|
1175
1437
|
return leases instanceof Map ? leases.get(leaseId) : leases?.[leaseId];
|
|
1176
1438
|
}
|
|
1177
1439
|
|
|
1440
|
+
function countLiveLeases(leases, nowMs) {
|
|
1441
|
+
let count = 0;
|
|
1442
|
+
const values = leases instanceof Map ? leases.values() : Object.values(leases);
|
|
1443
|
+
for (const lease of values) {
|
|
1444
|
+
if (Number.isFinite(lease?.expiresAt) && lease.expiresAt > nowMs) count += 1;
|
|
1445
|
+
}
|
|
1446
|
+
return count;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
function currentSharedLaneProvenance(value, leases, nowMs) {
|
|
1450
|
+
const evidence = sharedLaneEvidence(value);
|
|
1451
|
+
if (!evidence.waitCause || countLiveLeases(leases, nowMs) !== evidence.sharingCount) return {};
|
|
1452
|
+
return evidence.sharingOwnerLeaseIds.every((leaseId) => leaseFor(leases, leaseId)?.expiresAt > nowMs)
|
|
1453
|
+
? sharedLaneProvenance(evidence)
|
|
1454
|
+
: {};
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1178
1457
|
function reservationCost(reservation, resource, leases, nowMs) {
|
|
1179
1458
|
if (!reservation || ["cancelled", "reconciled"].includes(reservation.status)) return 0;
|
|
1180
1459
|
if (reservation.status === "scheduled") {
|
|
@@ -1187,11 +1466,13 @@ function reservationCost(reservation, resource, leases, nowMs) {
|
|
|
1187
1466
|
: reservation.costs;
|
|
1188
1467
|
const cost = chargedCosts?.[resource] ??
|
|
1189
1468
|
(reservation.resource === resource ? reservation.cost : 0);
|
|
1190
|
-
|
|
1469
|
+
const accounted = reservation.accountedCosts?.[resource] ?? 0;
|
|
1470
|
+
return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
|
|
1191
1471
|
}
|
|
1192
1472
|
|
|
1193
1473
|
function availableForGrant({
|
|
1194
1474
|
budget,
|
|
1475
|
+
resource = budget?.resource,
|
|
1195
1476
|
reservations = [],
|
|
1196
1477
|
leases = {},
|
|
1197
1478
|
nowMs,
|
|
@@ -1199,8 +1480,9 @@ function availableForGrant({
|
|
|
1199
1480
|
}) {
|
|
1200
1481
|
const normalized = normalizeBudgetResource(budget);
|
|
1201
1482
|
if (!normalized) return { mode: "probe", reason: "budget-unknown" };
|
|
1483
|
+
if (!RATE_RESOURCES.includes(resource)) return { mode: "probe", reason: "budget-resource" };
|
|
1202
1484
|
if (normalized.observedAt > nowMs) return { mode: "probe", reason: "budget-future" };
|
|
1203
|
-
if (nowMs - normalized.observedAt >
|
|
1485
|
+
if (nowMs - normalized.observedAt > budgetSnapshotTtl(resource)) {
|
|
1204
1486
|
return { mode: "probe", reason: "budget-stale" };
|
|
1205
1487
|
}
|
|
1206
1488
|
if (nowMs >= normalized.resetMs) return { mode: "probe", reason: "budget-reset" };
|
|
@@ -1226,7 +1508,7 @@ function availableForGrant({
|
|
|
1226
1508
|
|
|
1227
1509
|
const reserve = resourceReserve(normalized.limit);
|
|
1228
1510
|
const charged = chargedCost ?? reservations.reduce(
|
|
1229
|
-
(total, reservation) => total + reservationCost(reservation,
|
|
1511
|
+
(total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
|
|
1230
1512
|
0,
|
|
1231
1513
|
);
|
|
1232
1514
|
return {
|
|
@@ -1272,7 +1554,8 @@ function resourceDecision({
|
|
|
1272
1554
|
chargedCost = null,
|
|
1273
1555
|
}) {
|
|
1274
1556
|
const capacity = availableForGrant({
|
|
1275
|
-
budget
|
|
1557
|
+
budget,
|
|
1558
|
+
resource,
|
|
1276
1559
|
reservations,
|
|
1277
1560
|
leases,
|
|
1278
1561
|
nowMs,
|
|
@@ -1480,6 +1763,7 @@ function scheduleIntents({
|
|
|
1480
1763
|
const updatedCursors = { ...cursors };
|
|
1481
1764
|
const grants = [];
|
|
1482
1765
|
const denied = [];
|
|
1766
|
+
const liveLeaseCount = countLiveLeases(leases, nowMs);
|
|
1483
1767
|
const priorities = [...new Set(valid.map((intent) => intent.normalizedPriority))].sort((a, b) => a - b);
|
|
1484
1768
|
|
|
1485
1769
|
for (const priority of priorities) {
|
|
@@ -1510,9 +1794,9 @@ function scheduleIntents({
|
|
|
1510
1794
|
resource,
|
|
1511
1795
|
governorEpochPhaseAt(intent.phaseSeed, decisions[resource]),
|
|
1512
1796
|
]));
|
|
1797
|
+
const intrinsicAt = Math.max(nowMs, ...Object.values(phaseTimes));
|
|
1513
1798
|
const notBefore = Math.max(
|
|
1514
|
-
|
|
1515
|
-
...Object.values(phaseTimes),
|
|
1799
|
+
intrinsicAt,
|
|
1516
1800
|
...resources.map((resource) => updatedLanes[resource]?.nextAt ?? nowMs),
|
|
1517
1801
|
);
|
|
1518
1802
|
const expiring = resources.find((resource) => notBefore >= decisions[resource].resetMs);
|
|
@@ -1538,6 +1822,18 @@ function scheduleIntents({
|
|
|
1538
1822
|
status: "scheduled",
|
|
1539
1823
|
epochs: Object.fromEntries(resources.map((resource) => [resource, decisions[resource].epoch])),
|
|
1540
1824
|
};
|
|
1825
|
+
const limitingLanes = resources.filter((resource) =>
|
|
1826
|
+
updatedLanes[resource]?.nextAt === notBefore);
|
|
1827
|
+
const sharingOwnerLeaseIds = [...new Set(limitingLanes.map((resource) =>
|
|
1828
|
+
updatedCursors[resource]))];
|
|
1829
|
+
const sharedLane = liveLeaseCount > 1 && notBefore > intrinsicAt &&
|
|
1830
|
+
sharingOwnerLeaseIds.length > 0 && sharingOwnerLeaseIds.every((owner) =>
|
|
1831
|
+
owner !== intent.leaseId && leaseFor(leases, owner)?.expiresAt > nowMs);
|
|
1832
|
+
if (sharedLane) {
|
|
1833
|
+
reservation.waitCause = "shared-lane";
|
|
1834
|
+
reservation.sharingCount = liveLeaseCount;
|
|
1835
|
+
reservation.sharingOwnerLeaseIds = sharingOwnerLeaseIds;
|
|
1836
|
+
}
|
|
1541
1837
|
grants.push(reservation);
|
|
1542
1838
|
for (const resource of resources) {
|
|
1543
1839
|
chargedTotals[resource] += intent.costs[resource];
|
|
@@ -1562,7 +1858,11 @@ function scheduleIntents({
|
|
|
1562
1858
|
|
|
1563
1859
|
// ---------- Shared account governor ----------
|
|
1564
1860
|
|
|
1565
|
-
|
|
1861
|
+
// The scope version is deliberately stable. Stored protocol revisions must use
|
|
1862
|
+
// the same file so an older process sees the new exact shape and fails closed
|
|
1863
|
+
// instead of coordinating through a second file.
|
|
1864
|
+
const GOVERNOR_SCOPE_VERSION = 1;
|
|
1865
|
+
const GOVERNOR_STATE_VERSION = 2;
|
|
1566
1866
|
const GOVERNOR_MAX_LEASES = 128;
|
|
1567
1867
|
const GOVERNOR_MAX_INTENTS = 512;
|
|
1568
1868
|
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
@@ -1571,6 +1871,7 @@ const GOVERNOR_PROBE_DRAIN_MS = 30_000;
|
|
|
1571
1871
|
const GOVERNOR_PUBLICATION_REINSPECT_MS = 1_000;
|
|
1572
1872
|
const GOVERNOR_PROBE_TRANSITION_MS = 5_000;
|
|
1573
1873
|
const GOVERNOR_MAX_FUTURE_MS = 24 * 60 * 60 * 1000;
|
|
1874
|
+
const persistenceWaitCell = new Int32Array(new SharedArrayBuffer(4));
|
|
1574
1875
|
const GOVERNOR_OUTCOMES = new Set([
|
|
1575
1876
|
"measured-success",
|
|
1576
1877
|
"rejected",
|
|
@@ -1605,7 +1906,7 @@ function governorScopeHash(effectiveHost, authIdentity) {
|
|
|
1605
1906
|
const host = normalizeHost(effectiveHost);
|
|
1606
1907
|
if (!host || typeof authIdentity !== "string" || authIdentity.length === 0) return null;
|
|
1607
1908
|
return createHash("sha256")
|
|
1608
|
-
.update(JSON.stringify({ version:
|
|
1909
|
+
.update(JSON.stringify({ version: GOVERNOR_SCOPE_VERSION, effectiveHost: host, authCacheIdentity: authIdentity }))
|
|
1609
1910
|
.digest("hex");
|
|
1610
1911
|
}
|
|
1611
1912
|
|
|
@@ -1673,6 +1974,9 @@ function emptyGovernorState() {
|
|
|
1673
1974
|
version: GOVERNOR_STATE_VERSION,
|
|
1674
1975
|
epochs: { core: null, graphql: null },
|
|
1675
1976
|
budgets: {},
|
|
1977
|
+
observers: {
|
|
1978
|
+
core: { etag: null, outcome: "idle", at: 0, nextAt: 0 },
|
|
1979
|
+
},
|
|
1676
1980
|
probeClaim: null,
|
|
1677
1981
|
probeOutcome: { status: "idle", at: 0, nextAt: 0 },
|
|
1678
1982
|
leases: {},
|
|
@@ -1693,10 +1997,11 @@ function exactKeys(value, keys) {
|
|
|
1693
1997
|
keys.every((key) => Object.hasOwn(value, key));
|
|
1694
1998
|
}
|
|
1695
1999
|
|
|
1696
|
-
function normalizeGovernorBudget(raw, nowMs) {
|
|
2000
|
+
function normalizeGovernorBudget(raw, nowMs, resource = null) {
|
|
1697
2001
|
if (!exactKeys(raw, [
|
|
1698
2002
|
"limit", "remaining", "used", "resetMs", "observedAt", "blockUntil",
|
|
1699
2003
|
"blockReason", "laneNextAt", "roundRobinCursor", "lastExternalFactor", "epoch",
|
|
2004
|
+
"source", "factorBaseline", "knownLocalUsed",
|
|
1700
2005
|
])) return null;
|
|
1701
2006
|
const normalized = normalizeBudgetResource(raw);
|
|
1702
2007
|
if (
|
|
@@ -1708,6 +2013,14 @@ function normalizeGovernorBudget(raw, nowMs) {
|
|
|
1708
2013
|
(raw.blockReason !== null && !GOVERNOR_BLOCK_REASONS.has(raw.blockReason)) ||
|
|
1709
2014
|
(raw.roundRobinCursor !== null && !validGovernorId(raw.roundRobinCursor)) ||
|
|
1710
2015
|
!Number.isFinite(raw.lastExternalFactor) || raw.lastExternalFactor < 1 ||
|
|
2016
|
+
!["response-header", "core-observer", "rate-limit-probe"].includes(raw.source) ||
|
|
2017
|
+
raw.source === "rate-limit-probe" && resource === "core" ||
|
|
2018
|
+
!exactKeys(raw.factorBaseline, ["epoch", "used", "observedAt"]) ||
|
|
2019
|
+
!validGovernorEpoch(raw.factorBaseline.epoch) ||
|
|
2020
|
+
!Number.isSafeInteger(raw.factorBaseline.used) || raw.factorBaseline.used < 0 ||
|
|
2021
|
+
finiteTimestamp(raw.factorBaseline.observedAt, nowMs) === undefined ||
|
|
2022
|
+
raw.factorBaseline.observedAt > nowMs ||
|
|
2023
|
+
!Number.isSafeInteger(raw.knownLocalUsed) || raw.knownLocalUsed < 0 ||
|
|
1711
2024
|
!validGovernorEpoch(raw.epoch)
|
|
1712
2025
|
) return null;
|
|
1713
2026
|
return {
|
|
@@ -1718,6 +2031,9 @@ function normalizeGovernorBudget(raw, nowMs) {
|
|
|
1718
2031
|
roundRobinCursor: raw.roundRobinCursor,
|
|
1719
2032
|
lastExternalFactor: raw.lastExternalFactor,
|
|
1720
2033
|
epoch: raw.epoch,
|
|
2034
|
+
source: raw.source,
|
|
2035
|
+
factorBaseline: { ...raw.factorBaseline },
|
|
2036
|
+
knownLocalUsed: raw.knownLocalUsed,
|
|
1721
2037
|
};
|
|
1722
2038
|
}
|
|
1723
2039
|
|
|
@@ -1756,18 +2072,19 @@ function normalizeGovernorIntent(raw, nowMs) {
|
|
|
1756
2072
|
|
|
1757
2073
|
function normalizeGovernorReservation(raw, nowMs) {
|
|
1758
2074
|
if (!exactKeys(raw, [
|
|
1759
|
-
"leaseId", "intentId", "costs", "actualCosts", "notBefore", "status", "epochs",
|
|
2075
|
+
"leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
|
|
1760
2076
|
"startedAt", "completedAt", "outcome",
|
|
1761
2077
|
])) return null;
|
|
1762
2078
|
const costs = exactResourceCosts(raw.costs);
|
|
1763
2079
|
const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
|
|
2080
|
+
const accountedCosts = exactResourceCosts(raw.accountedCosts);
|
|
1764
2081
|
const epochs = exactKeys(raw.epochs, RATE_RESOURCES) &&
|
|
1765
2082
|
RATE_RESOURCES.every((resource) => raw.epochs[resource] === null || validGovernorEpoch(raw.epochs[resource]))
|
|
1766
2083
|
? { ...raw.epochs }
|
|
1767
2084
|
: null;
|
|
1768
2085
|
if (
|
|
1769
2086
|
!validGovernorId(raw.leaseId) || !validGovernorId(raw.intentId) ||
|
|
1770
|
-
!costs || actualCosts === null && raw.actualCosts !== null || !epochs ||
|
|
2087
|
+
!costs || actualCosts === null && raw.actualCosts !== null || !accountedCosts || !epochs ||
|
|
1771
2088
|
finiteTimestamp(raw.notBefore, nowMs) === undefined ||
|
|
1772
2089
|
!GOVERNOR_RESERVATION_STATUSES.has(raw.status) ||
|
|
1773
2090
|
finiteTimestamp(raw.startedAt, nowMs, { nullable: true }) === undefined ||
|
|
@@ -1777,23 +2094,42 @@ function normalizeGovernorReservation(raw, nowMs) {
|
|
|
1777
2094
|
(raw.outcome !== null && !GOVERNOR_OUTCOMES.has(raw.outcome))
|
|
1778
2095
|
) return null;
|
|
1779
2096
|
if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
|
|
1780
|
-
|
|
2097
|
+
if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
|
|
2098
|
+
return { ...raw, costs, actualCosts, accountedCosts, epochs };
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
function normalizeCoreObserver(raw, nowMs) {
|
|
2102
|
+
if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
|
|
2103
|
+
if (
|
|
2104
|
+
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
|
|
2105
|
+
!GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
|
|
2106
|
+
finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
|
|
2107
|
+
finiteTimestamp(raw.nextAt, nowMs) === undefined
|
|
2108
|
+
) return null;
|
|
2109
|
+
return { ...raw };
|
|
1781
2110
|
}
|
|
1782
2111
|
|
|
1783
2112
|
function normalizeProbeClaim(raw, nowMs) {
|
|
1784
2113
|
if (raw === null) return null;
|
|
1785
|
-
if (!exactKeys(raw, [
|
|
2114
|
+
if (!exactKeys(raw, [
|
|
2115
|
+
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds", "resources",
|
|
2116
|
+
])) {
|
|
1786
2117
|
return undefined;
|
|
1787
2118
|
}
|
|
2119
|
+
const resources = Array.isArray(raw.resources) && raw.resources.length > 0 &&
|
|
2120
|
+
raw.resources.every((resource) => RATE_RESOURCES.includes(resource))
|
|
2121
|
+
? [...new Set(raw.resources)]
|
|
2122
|
+
: null;
|
|
1788
2123
|
if (
|
|
1789
2124
|
!validGovernorId(raw.ownerLeaseId) || !validGovernorId(raw.nonce) ||
|
|
1790
2125
|
finiteTimestamp(raw.leaseUntil, nowMs) === undefined ||
|
|
1791
2126
|
finiteTimestamp(raw.nextAt, nowMs) === undefined ||
|
|
1792
2127
|
finiteTimestamp(raw.claimAt, nowMs) === undefined || raw.claimAt > nowMs ||
|
|
1793
2128
|
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
1794
|
-
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
2129
|
+
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12))) ||
|
|
2130
|
+
!resources
|
|
1795
2131
|
) return undefined;
|
|
1796
|
-
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
2132
|
+
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)], resources };
|
|
1797
2133
|
}
|
|
1798
2134
|
|
|
1799
2135
|
function normalizeManualProbe(raw, nowMs) {
|
|
@@ -1810,13 +2146,14 @@ function normalizeManualProbe(raw, nowMs) {
|
|
|
1810
2146
|
|
|
1811
2147
|
function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
1812
2148
|
if (!exactKeys(raw, [
|
|
1813
|
-
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
2149
|
+
"version", "epochs", "budgets", "observers", "probeClaim", "probeOutcome", "leases",
|
|
1814
2150
|
"intents", "reservations", "manualProbe",
|
|
1815
2151
|
]) || raw.version !== GOVERNOR_STATE_VERSION) return null;
|
|
1816
2152
|
if (
|
|
1817
2153
|
!exactKeys(raw.epochs, RATE_RESOURCES) ||
|
|
1818
2154
|
RATE_RESOURCES.some((resource) => raw.epochs[resource] !== null && !validGovernorEpoch(raw.epochs[resource])) ||
|
|
1819
2155
|
!isRecord(raw.budgets) || Object.keys(raw.budgets).some((resource) => !RATE_RESOURCES.includes(resource)) ||
|
|
2156
|
+
!exactKeys(raw.observers, ["core"]) ||
|
|
1820
2157
|
!isRecord(raw.leases) || !isRecord(raw.intents) || !isRecord(raw.reservations) ||
|
|
1821
2158
|
!exactKeys(raw.probeOutcome, ["status", "at", "nextAt"]) ||
|
|
1822
2159
|
!GOVERNOR_PROBE_STATUSES.has(raw.probeOutcome.status) ||
|
|
@@ -1828,8 +2165,10 @@ function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
|
1828
2165
|
const state = emptyGovernorState();
|
|
1829
2166
|
state.epochs = { ...raw.epochs };
|
|
1830
2167
|
state.probeOutcome = { ...raw.probeOutcome };
|
|
2168
|
+
state.observers.core = normalizeCoreObserver(raw.observers.core, nowMs);
|
|
2169
|
+
if (!state.observers.core) return null;
|
|
1831
2170
|
for (const resource of Object.keys(raw.budgets)) {
|
|
1832
|
-
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs);
|
|
2171
|
+
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs, resource);
|
|
1833
2172
|
if (!budget) return null;
|
|
1834
2173
|
state.budgets[resource] = budget;
|
|
1835
2174
|
}
|
|
@@ -1869,7 +2208,36 @@ function serializeGovernorState(state) {
|
|
|
1869
2208
|
return `${JSON.stringify(state)}\n`;
|
|
1870
2209
|
}
|
|
1871
2210
|
|
|
1872
|
-
function
|
|
2211
|
+
function migrateGovernorState(raw, nowMs) {
|
|
2212
|
+
if (!exactKeys(raw, [
|
|
2213
|
+
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
2214
|
+
"intents", "reservations", "manualProbe",
|
|
2215
|
+
]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
|
|
2216
|
+
const migrated = structuredClone(raw);
|
|
2217
|
+
migrated.version = GOVERNOR_STATE_VERSION;
|
|
2218
|
+
migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
|
|
2219
|
+
// The v1 core value came from /rate_limit, which is not authoritative. Do not
|
|
2220
|
+
// preserve it as bootstrap evidence. GraphQL remains valid.
|
|
2221
|
+
delete migrated.budgets.core;
|
|
2222
|
+
migrated.epochs.core = null;
|
|
2223
|
+
if (migrated.budgets.graphql) {
|
|
2224
|
+
const budget = migrated.budgets.graphql;
|
|
2225
|
+
budget.source = "rate-limit-probe";
|
|
2226
|
+
budget.factorBaseline = {
|
|
2227
|
+
epoch: budget.epoch,
|
|
2228
|
+
used: budget.used,
|
|
2229
|
+
observedAt: budget.observedAt,
|
|
2230
|
+
};
|
|
2231
|
+
budget.knownLocalUsed = 0;
|
|
2232
|
+
}
|
|
2233
|
+
for (const reservation of Object.values(migrated.reservations)) {
|
|
2234
|
+
reservation.accountedCosts = { core: 0, graphql: 0 };
|
|
2235
|
+
}
|
|
2236
|
+
if (migrated.probeClaim) migrated.probeClaim.resources = [...RATE_RESOURCES];
|
|
2237
|
+
return normalizeGovernorState(migrated, nowMs);
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
1873
2241
|
let raw;
|
|
1874
2242
|
try {
|
|
1875
2243
|
raw = readFileSync(path, "utf8");
|
|
@@ -1879,8 +2247,16 @@ function readGovernorState(path, nowMs) {
|
|
|
1879
2247
|
: { ok: false, reason: "unwritable" };
|
|
1880
2248
|
}
|
|
1881
2249
|
try {
|
|
1882
|
-
const
|
|
1883
|
-
|
|
2250
|
+
const parsed = JSON.parse(raw);
|
|
2251
|
+
const normalized = normalizeGovernorState(parsed, nowMs);
|
|
2252
|
+
if (normalized) return { ok: true, value: normalized };
|
|
2253
|
+
const migrated = migrateGovernorState(parsed, nowMs);
|
|
2254
|
+
if (!migrated) return { ok: false, reason: "corrupt" };
|
|
2255
|
+
if (!persistMigration) return { ok: true, value: migrated, migrated: true };
|
|
2256
|
+
// Read-only callers persist the exact migration while they still hold the
|
|
2257
|
+
// scope lock. Mutating callers use this same value for their final write.
|
|
2258
|
+
const written = writeGovernorState(path, migrated);
|
|
2259
|
+
return written.ok ? { ok: true, value: migrated, migrated: true } : written;
|
|
1884
2260
|
} catch {
|
|
1885
2261
|
return { ok: false, reason: "corrupt" };
|
|
1886
2262
|
}
|
|
@@ -2109,13 +2485,21 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
2109
2485
|
// held, immediately before it reads and mutates the shared state.
|
|
2110
2486
|
const current = currentGovernorScope(scope);
|
|
2111
2487
|
if (!current.ok) return current;
|
|
2112
|
-
const loaded = readGovernorState(scope.path, at);
|
|
2488
|
+
const loaded = readGovernorState(scope.path, at, { persistMigration: false });
|
|
2113
2489
|
if (!loaded.ok) return loaded;
|
|
2114
2490
|
const state = loaded.value;
|
|
2491
|
+
// A mutator can reject without writing. Keep an untouched migration so
|
|
2492
|
+
// that even that path upgrades the file once before releasing the lock.
|
|
2493
|
+
const migration = loaded.migrated ? structuredClone(state) : null;
|
|
2494
|
+
const returnAfterMigration = (result) => {
|
|
2495
|
+
if (migration === null) return result;
|
|
2496
|
+
const written = writeGovernorState(scope.path, migration);
|
|
2497
|
+
return written.ok ? result : written;
|
|
2498
|
+
};
|
|
2115
2499
|
const value = mutate(state, at);
|
|
2116
|
-
if (value?.ok === false && value.write !== true) return value;
|
|
2500
|
+
if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
|
|
2117
2501
|
const normalized = normalizeGovernorState(state, at);
|
|
2118
|
-
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
2502
|
+
if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
|
|
2119
2503
|
const written = writeGovernorState(scope.path, normalized);
|
|
2120
2504
|
if (!written.ok) return written;
|
|
2121
2505
|
return value?.ok === false ? { ok: false, reason: value.reason } : { ok: true, value: value?.value };
|
|
@@ -2168,6 +2552,7 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
2168
2552
|
intentId: grant.intentId,
|
|
2169
2553
|
costs: grant.costs,
|
|
2170
2554
|
actualCosts: null,
|
|
2555
|
+
accountedCosts: { core: 0, graphql: 0 },
|
|
2171
2556
|
notBefore: grant.notBefore,
|
|
2172
2557
|
status: "scheduled",
|
|
2173
2558
|
epochs: { core: grant.epochs.core ?? null, graphql: grant.epochs.graphql ?? null },
|
|
@@ -2246,14 +2631,38 @@ function claimProbe(scope, leaseId, nowMs) {
|
|
|
2246
2631
|
if (state.probeClaim && state.probeClaim.leaseUntil > at) {
|
|
2247
2632
|
return { value: { status: "waiting", leaseUntil: state.probeClaim.leaseUntil } };
|
|
2248
2633
|
}
|
|
2249
|
-
const
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
const
|
|
2634
|
+
const coreResetAt = Number.isFinite(state.budgets.core?.resetMs)
|
|
2635
|
+
? state.budgets.core.resetMs + BUDGET_RESET_GRACE_MS
|
|
2636
|
+
: Number.POSITIVE_INFINITY;
|
|
2637
|
+
const graphqlResetAt = Number.isFinite(state.budgets.graphql?.resetMs)
|
|
2638
|
+
? state.budgets.graphql.resetMs + BUDGET_RESET_GRACE_MS
|
|
2639
|
+
: Number.POSITIVE_INFINITY;
|
|
2640
|
+
const dueAt = {
|
|
2641
|
+
core: Math.min(
|
|
2642
|
+
state.observers.core.nextAt,
|
|
2643
|
+
coreResetAt,
|
|
2644
|
+
),
|
|
2645
|
+
graphql: Math.min(
|
|
2646
|
+
state.probeOutcome.nextAt,
|
|
2647
|
+
graphqlResetAt,
|
|
2648
|
+
// A core reset starts a new shared accounting epoch. Refresh the free
|
|
2649
|
+
// GraphQL counter in that same nonce so external-spend baselines and
|
|
2650
|
+
// the protected one-publication reset contract advance together.
|
|
2651
|
+
coreResetAt,
|
|
2652
|
+
),
|
|
2653
|
+
};
|
|
2654
|
+
const nextAt = Math.min(...Object.values(dueAt));
|
|
2253
2655
|
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
2656
|
+
// One nonce owns the control-plane transition, but only the sources whose
|
|
2657
|
+
// persisted clocks are due may be read. In particular, a GraphQL minute
|
|
2658
|
+
// sample cannot spend or retry the held core observer before its reset.
|
|
2659
|
+
const resources = RATE_RESOURCES.filter((resource) => dueAt[resource] <= at);
|
|
2254
2660
|
const nonce = randomUUID();
|
|
2255
2661
|
const startedReservationIds = Object.entries(state.reservations)
|
|
2256
|
-
|
|
2662
|
+
// An expired owner can no longer settle its request. Keep that uncertain
|
|
2663
|
+
// cost charged, but do not make every later probe wait for it to finish.
|
|
2664
|
+
.filter(([, reservation]) => reservation.status === "started" &&
|
|
2665
|
+
state.leases[reservation.leaseId]?.expiresAt > at)
|
|
2257
2666
|
.map(([id]) => id);
|
|
2258
2667
|
state.probeClaim = {
|
|
2259
2668
|
ownerLeaseId: leaseId,
|
|
@@ -2262,9 +2671,24 @@ function claimProbe(scope, leaseId, nowMs) {
|
|
|
2262
2671
|
nextAt: at,
|
|
2263
2672
|
claimAt: at,
|
|
2264
2673
|
startedReservationIds,
|
|
2674
|
+
resources,
|
|
2265
2675
|
};
|
|
2266
|
-
|
|
2267
|
-
|
|
2676
|
+
if (resources.includes("core")) {
|
|
2677
|
+
state.observers.core.outcome = "waiting";
|
|
2678
|
+
state.observers.core.at = at;
|
|
2679
|
+
state.observers.core.nextAt = at + GOVERNOR_PROBE_LEASE_MS;
|
|
2680
|
+
}
|
|
2681
|
+
if (resources.includes("graphql")) {
|
|
2682
|
+
state.probeOutcome = { status: "waiting", at, nextAt: at + GOVERNOR_PROBE_LEASE_MS };
|
|
2683
|
+
}
|
|
2684
|
+
return { value: {
|
|
2685
|
+
status: "claimed",
|
|
2686
|
+
nonce,
|
|
2687
|
+
leaseUntil: state.probeClaim.leaseUntil,
|
|
2688
|
+
startedReservationIds,
|
|
2689
|
+
resources,
|
|
2690
|
+
coreEtag: state.observers.core.etag,
|
|
2691
|
+
} };
|
|
2268
2692
|
});
|
|
2269
2693
|
}
|
|
2270
2694
|
|
|
@@ -2279,18 +2703,50 @@ function renewProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
|
2279
2703
|
});
|
|
2280
2704
|
}
|
|
2281
2705
|
|
|
2282
|
-
function
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2706
|
+
function budgetFromObservation(raw, previous, nowMs, {
|
|
2707
|
+
resource,
|
|
2708
|
+
source,
|
|
2709
|
+
receivedAt = nowMs,
|
|
2710
|
+
clearProvenBlock = false,
|
|
2711
|
+
allowEpochChange = false,
|
|
2712
|
+
} = {}) {
|
|
2713
|
+
let normalized = normalizeBudgetResource({ ...raw, observedAt: receivedAt });
|
|
2714
|
+
if (
|
|
2715
|
+
!normalized || normalized.resetMs <= nowMs || receivedAt > nowMs ||
|
|
2716
|
+
!RATE_RESOURCES.includes(resource) ||
|
|
2717
|
+
!["response-header", "core-observer", "rate-limit-probe"].includes(source) ||
|
|
2718
|
+
resource === "core" && source === "rate-limit-probe"
|
|
2719
|
+
) return { status: "invalid" };
|
|
2720
|
+
const epoch = `${normalized.limit}:${normalized.resetMs}`;
|
|
2721
|
+
// Endpoint responses can carry cached or endpoint-specific rate headers.
|
|
2722
|
+
// They may refine the counter inside the epoch established by its claimed
|
|
2723
|
+
// observer, but only that observer may move the resource into a new epoch.
|
|
2724
|
+
if (!allowEpochChange && (!previous || previous.epoch !== epoch)) {
|
|
2725
|
+
return { status: "ignored", epoch };
|
|
2726
|
+
}
|
|
2727
|
+
if (previous) {
|
|
2728
|
+
if (normalized.resetMs < previous.resetMs) return { status: "ignored" };
|
|
2729
|
+
if (budgetEpoch(previous) === epoch && normalized.used < previous.used) {
|
|
2730
|
+
if (!allowEpochChange) return { status: "ignored" };
|
|
2731
|
+
// The claimed observer can lag a newer endpoint response within the
|
|
2732
|
+
// same GitHub window. Refresh its timestamp without giving that stale
|
|
2733
|
+
// counter any capacity: retain the higher used and lower remaining
|
|
2734
|
+
// values until the observer catches up.
|
|
2735
|
+
normalized = {
|
|
2736
|
+
...normalized,
|
|
2737
|
+
used: previous.used,
|
|
2738
|
+
remaining: Math.min(normalized.remaining, previous.remaining),
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
if (
|
|
2742
|
+
normalized.resetMs === previous.resetMs && normalized.used === previous.used &&
|
|
2743
|
+
receivedAt <= previous.observedAt
|
|
2744
|
+
) return { status: "ignored" };
|
|
2745
|
+
}
|
|
2291
2746
|
const epochChanged = !previous || previous.epoch !== epoch;
|
|
2292
|
-
const
|
|
2293
|
-
|
|
2747
|
+
const preserveBlock = !clearProvenBlock || !epochChanged;
|
|
2748
|
+
const blockActive = preserveBlock && Number.isFinite(previous?.blockUntil) && previous.blockUntil > nowMs;
|
|
2749
|
+
const next = {
|
|
2294
2750
|
...normalized,
|
|
2295
2751
|
blockUntil: blockActive ? previous.blockUntil : null,
|
|
2296
2752
|
blockReason: blockActive ? previous.blockReason : null,
|
|
@@ -2298,7 +2754,13 @@ function budgetFromProbe(raw, previous, nowMs) {
|
|
|
2298
2754
|
roundRobinCursor: previous?.roundRobinCursor ?? null,
|
|
2299
2755
|
lastExternalFactor: previous?.lastExternalFactor ?? 1,
|
|
2300
2756
|
epoch,
|
|
2757
|
+
source,
|
|
2758
|
+
factorBaseline: epochChanged || !previous?.factorBaseline
|
|
2759
|
+
? { epoch, used: normalized.used, observedAt: receivedAt }
|
|
2760
|
+
: { ...previous.factorBaseline },
|
|
2761
|
+
knownLocalUsed: epochChanged ? 0 : previous.knownLocalUsed,
|
|
2301
2762
|
};
|
|
2763
|
+
return { status: "accepted", budget: next, epochChanged };
|
|
2302
2764
|
}
|
|
2303
2765
|
|
|
2304
2766
|
function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
@@ -2307,45 +2769,103 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2307
2769
|
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2308
2770
|
return { ok: false, reason: "stale" };
|
|
2309
2771
|
}
|
|
2310
|
-
const nextBudgets = {};
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2772
|
+
const nextBudgets = { ...state.budgets };
|
|
2773
|
+
const nextEpochs = { ...state.epochs };
|
|
2774
|
+
const changedResources = [];
|
|
2775
|
+
const resetResources = [];
|
|
2776
|
+
if (!claim.resources.every((resource) => budgets?.[resource])) {
|
|
2777
|
+
return { ok: false, reason: "corrupt" };
|
|
2314
2778
|
}
|
|
2315
|
-
const
|
|
2316
|
-
resource
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2779
|
+
for (const resource of claim.resources) {
|
|
2780
|
+
const supplied = budgets?.[resource];
|
|
2781
|
+
if (!supplied) continue;
|
|
2782
|
+
const raw = supplied.budget ?? supplied;
|
|
2783
|
+
const source = supplied.source ?? (resource === "core" ? "core-observer" : "rate-limit-probe");
|
|
2784
|
+
const ownerSource = resource === "core" ? "core-observer" : "rate-limit-probe";
|
|
2785
|
+
if (source !== ownerSource) return { ok: false, reason: "corrupt" };
|
|
2786
|
+
const observed = budgetFromObservation(raw, state.budgets[resource], at, {
|
|
2787
|
+
resource,
|
|
2788
|
+
source,
|
|
2789
|
+
receivedAt: supplied.receivedAt ?? at,
|
|
2790
|
+
clearProvenBlock: true,
|
|
2791
|
+
allowEpochChange: true,
|
|
2792
|
+
});
|
|
2793
|
+
if (observed.status === "invalid") return { ok: false, reason: "corrupt" };
|
|
2794
|
+
if (observed.status === "ignored") continue;
|
|
2795
|
+
nextBudgets[resource] = observed.budget;
|
|
2796
|
+
if (resource === "core" && supplied.blocked === true) {
|
|
2797
|
+
nextBudgets[resource].blockUntil = observed.budget.resetMs;
|
|
2798
|
+
nextBudgets[resource].blockReason = "rate-limit";
|
|
2799
|
+
}
|
|
2800
|
+
nextEpochs[resource] = observed.budget.epoch;
|
|
2801
|
+
changedResources.push(resource);
|
|
2802
|
+
if (observed.epochChanged && state.epochs[resource] !== null) resetResources.push(resource);
|
|
2803
|
+
}
|
|
2804
|
+
if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
|
|
2805
|
+
const completedBeforeClaim = Object.entries(state.reservations)
|
|
2806
|
+
.filter(([, reservation]) =>
|
|
2807
|
+
reservation.status === "completed" && reservation.completedAt < claim.claimAt);
|
|
2808
|
+
for (const resource of changedResources) {
|
|
2320
2809
|
const previous = state.budgets[resource];
|
|
2321
|
-
if (!previous || nextBudgets[resource].
|
|
2322
|
-
const
|
|
2323
|
-
|
|
2324
|
-
|
|
2810
|
+
if (!previous || nextBudgets[resource].epoch !== previous.epoch) continue;
|
|
2811
|
+
const baseline = previous.factorBaseline;
|
|
2812
|
+
if (!baseline || baseline.epoch !== nextBudgets[resource].epoch || nextBudgets[resource].used < baseline.used) {
|
|
2813
|
+
continue;
|
|
2814
|
+
}
|
|
2815
|
+
const legacyCompletedDelta = completedBeforeClaim
|
|
2816
|
+
.reduce((total, [, reservation]) =>
|
|
2817
|
+
total + (reservation.completedAt > baseline.observedAt
|
|
2818
|
+
? reservationCost(reservation, resource, state.leases, at)
|
|
2819
|
+
: 0), 0);
|
|
2325
2820
|
const factor = nextExternalFactor({
|
|
2326
2821
|
lastExternalFactor: previous.lastExternalFactor,
|
|
2327
|
-
globalUsedDelta: nextBudgets[resource].used -
|
|
2328
|
-
sharedCompletedDelta,
|
|
2822
|
+
globalUsedDelta: nextBudgets[resource].used - baseline.used,
|
|
2823
|
+
sharedCompletedDelta: previous.knownLocalUsed + legacyCompletedDelta,
|
|
2329
2824
|
});
|
|
2330
2825
|
if (factor === null) return { ok: false, reason: "corrupt" };
|
|
2331
2826
|
nextBudgets[resource].lastExternalFactor = factor;
|
|
2827
|
+
nextBudgets[resource].factorBaseline = {
|
|
2828
|
+
epoch: nextBudgets[resource].epoch,
|
|
2829
|
+
used: nextBudgets[resource].used,
|
|
2830
|
+
observedAt: nextBudgets[resource].observedAt,
|
|
2831
|
+
};
|
|
2832
|
+
nextBudgets[resource].knownLocalUsed = 0;
|
|
2332
2833
|
}
|
|
2333
|
-
const
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
if (
|
|
2338
|
-
delete state.reservations[id];
|
|
2339
|
-
}
|
|
2834
|
+
for (const [id, reservation] of completedBeforeClaim) {
|
|
2835
|
+
const accountedByEveryResource = RATE_RESOURCES.every((resource) =>
|
|
2836
|
+
reservationCost(reservation, resource, state.leases, at) === 0 ||
|
|
2837
|
+
nextBudgets[resource]?.factorBaseline?.observedAt >= reservation.completedAt);
|
|
2838
|
+
if (accountedByEveryResource) delete state.reservations[id];
|
|
2340
2839
|
}
|
|
2341
2840
|
state.budgets = nextBudgets;
|
|
2342
2841
|
state.epochs = nextEpochs;
|
|
2343
2842
|
state.probeClaim = null;
|
|
2344
|
-
|
|
2345
|
-
|
|
2843
|
+
if (claim.resources.includes("core")) {
|
|
2844
|
+
const core = nextBudgets.core;
|
|
2845
|
+
state.observers.core = {
|
|
2846
|
+
etag: typeof budgets.core.etag === "string" ? budgets.core.etag : state.observers.core.etag,
|
|
2847
|
+
outcome: "healthy",
|
|
2848
|
+
at,
|
|
2849
|
+
nextAt: core.remaining === 0
|
|
2850
|
+
? core.resetMs + BUDGET_RESET_GRACE_MS
|
|
2851
|
+
: at + BUDGET_PROBE_MS,
|
|
2852
|
+
};
|
|
2853
|
+
}
|
|
2854
|
+
if (claim.resources.includes("graphql")) {
|
|
2855
|
+
state.probeOutcome = {
|
|
2856
|
+
status: "healthy",
|
|
2857
|
+
at,
|
|
2858
|
+
nextAt: at + BUDGET_PROBE_MS,
|
|
2859
|
+
};
|
|
2860
|
+
}
|
|
2861
|
+
if (resetResources.length > 0) state.manualProbe = null;
|
|
2346
2862
|
else if (state.manualProbe &&
|
|
2347
2863
|
Object.values(nextEpochs).includes(state.manualProbe.requestedEpoch) &&
|
|
2348
|
-
RATE_RESOURCES.some((resource) => availableForGrant({
|
|
2864
|
+
RATE_RESOURCES.some((resource) => availableForGrant({
|
|
2865
|
+
budget: nextBudgets[resource],
|
|
2866
|
+
resource,
|
|
2867
|
+
nowMs: at,
|
|
2868
|
+
}).spendable === 0)) {
|
|
2349
2869
|
state.manualProbe.satisfiedAt = at;
|
|
2350
2870
|
}
|
|
2351
2871
|
scheduleGovernorState(state, at);
|
|
@@ -2360,8 +2880,18 @@ function failProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
|
2360
2880
|
return { ok: false, reason: "stale" };
|
|
2361
2881
|
}
|
|
2362
2882
|
state.probeClaim = null;
|
|
2363
|
-
|
|
2364
|
-
|
|
2883
|
+
if (claim.resources.includes("graphql")) {
|
|
2884
|
+
state.probeOutcome = { status: "failed", at, nextAt: at + BUDGET_PROBE_MS };
|
|
2885
|
+
}
|
|
2886
|
+
if (claim.resources.includes("core")) {
|
|
2887
|
+
state.observers.core = {
|
|
2888
|
+
...state.observers.core,
|
|
2889
|
+
outcome: "failed",
|
|
2890
|
+
at,
|
|
2891
|
+
nextAt: at + BUDGET_PROBE_MS,
|
|
2892
|
+
};
|
|
2893
|
+
}
|
|
2894
|
+
return { value: { retryAt: at + BUDGET_PROBE_MS } };
|
|
2365
2895
|
});
|
|
2366
2896
|
}
|
|
2367
2897
|
|
|
@@ -2374,7 +2904,16 @@ function requestManualProbe(scope, leaseId, epoch, observedAt, nowMs) {
|
|
|
2374
2904
|
return { value: { status: state.manualProbe.satisfiedAt === null ? "pending" : "satisfied", ...state.manualProbe } };
|
|
2375
2905
|
}
|
|
2376
2906
|
state.manualProbe = { requestedEpoch: epoch, baselineObservedAt: observedAt, satisfiedAt: null };
|
|
2377
|
-
|
|
2907
|
+
if (state.epochs.core === epoch) {
|
|
2908
|
+
const core = state.budgets.core;
|
|
2909
|
+
const heldThroughReset = core?.blockUntil > at && core.blockUntil === core.resetMs;
|
|
2910
|
+
state.observers.core.nextAt = heldThroughReset
|
|
2911
|
+
? Math.max(state.observers.core.nextAt, core.blockUntil + BUDGET_RESET_GRACE_MS)
|
|
2912
|
+
: Math.min(state.observers.core.nextAt || at, at);
|
|
2913
|
+
}
|
|
2914
|
+
if (state.epochs.graphql === epoch) {
|
|
2915
|
+
state.probeOutcome.nextAt = Math.min(state.probeOutcome.nextAt || at, at);
|
|
2916
|
+
}
|
|
2378
2917
|
return { value: { status: "pending", ...state.manualProbe } };
|
|
2379
2918
|
});
|
|
2380
2919
|
}
|
|
@@ -2410,9 +2949,15 @@ function registerIntent(scope, intent) {
|
|
|
2410
2949
|
state.intents[intent.id] = normalized;
|
|
2411
2950
|
const scheduled = scheduleGovernorState(state, nowMs);
|
|
2412
2951
|
const reservation = state.reservations[reservationId];
|
|
2952
|
+
const grant = scheduled.grants.find((item) => item.id === reservationId);
|
|
2413
2953
|
const denial = scheduled.denied.find((item) => item.intentId === intent.id);
|
|
2414
2954
|
return { value: reservation
|
|
2415
|
-
? {
|
|
2955
|
+
? {
|
|
2956
|
+
status: "scheduled",
|
|
2957
|
+
reservationId,
|
|
2958
|
+
...reservation,
|
|
2959
|
+
...sharedLaneEvidence(grant),
|
|
2960
|
+
}
|
|
2416
2961
|
: {
|
|
2417
2962
|
status: denial?.mode ?? "pending",
|
|
2418
2963
|
intentId: intent.id,
|
|
@@ -2430,7 +2975,13 @@ function readIntentDecision(scope, intentId, nowMs) {
|
|
|
2430
2975
|
const scheduled = scheduleGovernorState(state, at);
|
|
2431
2976
|
const reservationId = `reservation:${intentId}`;
|
|
2432
2977
|
const reservation = state.reservations[reservationId];
|
|
2433
|
-
|
|
2978
|
+
const grant = scheduled.grants.find((item) => item.id === reservationId);
|
|
2979
|
+
if (reservation) return { value: {
|
|
2980
|
+
status: reservation.status,
|
|
2981
|
+
reservationId,
|
|
2982
|
+
...reservation,
|
|
2983
|
+
...sharedLaneEvidence(grant),
|
|
2984
|
+
} };
|
|
2434
2985
|
if (!state.intents[intentId]) return { ok: false, reason: "stale" };
|
|
2435
2986
|
const denial = scheduled.denied.find((item) => item.intentId === intentId);
|
|
2436
2987
|
return { value: {
|
|
@@ -2476,7 +3027,7 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
2476
3027
|
for (const resource of RATE_RESOURCES.filter((name) => reservation.costs[name] > 0)) {
|
|
2477
3028
|
const budget = state.budgets[resource];
|
|
2478
3029
|
if (
|
|
2479
|
-
!budget || at - budget.observedAt >
|
|
3030
|
+
!budget || at - budget.observedAt > budgetSnapshotTtl(resource) || at >= budget.resetMs ||
|
|
2480
3031
|
state.epochs[resource] !== reservation.epochs[resource] ||
|
|
2481
3032
|
(Number.isFinite(budget.blockUntil) && budget.blockUntil > at)
|
|
2482
3033
|
) {
|
|
@@ -2514,10 +3065,107 @@ function completeReservation(scope, reservationId, completion, nowMs) {
|
|
|
2514
3065
|
reservation.completedAt = at;
|
|
2515
3066
|
reservation.outcome = completion.outcome;
|
|
2516
3067
|
reservation.actualCosts = { ...measured };
|
|
3068
|
+
reservation.accountedCosts = { core: 0, graphql: 0 };
|
|
2517
3069
|
return { value: { status: "completed", actualCosts: reservation.actualCosts } };
|
|
2518
3070
|
});
|
|
2519
3071
|
}
|
|
2520
3072
|
|
|
3073
|
+
function settleReservationWithBudgetObservations(
|
|
3074
|
+
scope,
|
|
3075
|
+
leaseId,
|
|
3076
|
+
reservationId,
|
|
3077
|
+
completion,
|
|
3078
|
+
nowMs,
|
|
3079
|
+
) {
|
|
3080
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
3081
|
+
const reservation = state.reservations[reservationId];
|
|
3082
|
+
const lease = state.leases[leaseId];
|
|
3083
|
+
if (
|
|
3084
|
+
!lease || lease.expiresAt <= at || !reservation ||
|
|
3085
|
+
reservation.leaseId !== leaseId || reservation.status !== "started" ||
|
|
3086
|
+
!GOVERNOR_OUTCOMES.has(completion?.outcome)
|
|
3087
|
+
) return { ok: false, reason: "stale" };
|
|
3088
|
+
const measured = completion.outcome === "measured-success"
|
|
3089
|
+
? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
|
|
3090
|
+
: reservation.costs;
|
|
3091
|
+
if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
|
|
3092
|
+
return { ok: false, reason: "corrupt" };
|
|
3093
|
+
}
|
|
3094
|
+
const observations = Array.isArray(completion.observations) ? completion.observations : [];
|
|
3095
|
+
if (observations.some((observation) =>
|
|
3096
|
+
!isRecord(observation) || observation.source !== "response-header" ||
|
|
3097
|
+
!Number.isFinite(observation.receivedAt) || observation.receivedAt > at)) {
|
|
3098
|
+
return { ok: false, reason: "corrupt" };
|
|
3099
|
+
}
|
|
3100
|
+
const acceptedCosts = { core: 0, graphql: 0 };
|
|
3101
|
+
const acceptedAfterBaseline = { core: 0, graphql: 0 };
|
|
3102
|
+
for (const resource of RATE_RESOURCES) {
|
|
3103
|
+
const candidates = observations
|
|
3104
|
+
.filter((observation) => observation.resource === resource)
|
|
3105
|
+
.sort((left, right) =>
|
|
3106
|
+
left.resetMs - right.resetMs || left.used - right.used || left.receivedAt - right.receivedAt);
|
|
3107
|
+
for (const observation of candidates) {
|
|
3108
|
+
const observed = budgetFromObservation(observation, state.budgets[resource], at, {
|
|
3109
|
+
resource,
|
|
3110
|
+
source: observation.source,
|
|
3111
|
+
receivedAt: observation.receivedAt,
|
|
3112
|
+
clearProvenBlock: false,
|
|
3113
|
+
});
|
|
3114
|
+
if (observed.status === "invalid") continue;
|
|
3115
|
+
const definiteCost = Number.isSafeInteger(observation.cost) && observation.cost >= 0
|
|
3116
|
+
? observation.cost
|
|
3117
|
+
: 0;
|
|
3118
|
+
if (observed.status === "accepted") {
|
|
3119
|
+
state.budgets[resource] = observed.budget;
|
|
3120
|
+
state.epochs[resource] = observed.budget.epoch;
|
|
3121
|
+
acceptedCosts[resource] += definiteCost;
|
|
3122
|
+
if (observation.receivedAt > observed.budget.factorBaseline.observedAt) {
|
|
3123
|
+
acceptedAfterBaseline[resource] += definiteCost;
|
|
3124
|
+
}
|
|
3125
|
+
} else if (
|
|
3126
|
+
observation.source === "response-header" &&
|
|
3127
|
+
state.budgets[resource]?.epoch === `${observation.limit}:${observation.resetMs}` &&
|
|
3128
|
+
state.budgets[resource].used >= observation.used
|
|
3129
|
+
) {
|
|
3130
|
+
// A newer same-window counter already includes this completed call.
|
|
3131
|
+
acceptedCosts[resource] += definiteCost;
|
|
3132
|
+
if (observation.receivedAt > state.budgets[resource].factorBaseline.observedAt) {
|
|
3133
|
+
acceptedAfterBaseline[resource] += definiteCost;
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
reservation.status = "completed";
|
|
3139
|
+
reservation.completedAt = at;
|
|
3140
|
+
reservation.outcome = completion.outcome;
|
|
3141
|
+
reservation.actualCosts = { ...measured };
|
|
3142
|
+
reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3143
|
+
resource,
|
|
3144
|
+
completion.outcome === "measured-success"
|
|
3145
|
+
? Math.min(measured[resource], acceptedCosts[resource])
|
|
3146
|
+
: 0,
|
|
3147
|
+
]));
|
|
3148
|
+
for (const resource of RATE_RESOURCES) {
|
|
3149
|
+
const budget = state.budgets[resource];
|
|
3150
|
+
if (!budget || completion.outcome !== "measured-success") continue;
|
|
3151
|
+
// accountedCosts removes the portion already present in an authoritative
|
|
3152
|
+
// counter from the reservation residual. knownLocalUsed contains only
|
|
3153
|
+
// that same portion when it arrived after the factor baseline. Their sum
|
|
3154
|
+
// is therefore exactly the measured local cost, never twice that cost.
|
|
3155
|
+
budget.knownLocalUsed += Math.min(
|
|
3156
|
+
reservation.accountedCosts[resource],
|
|
3157
|
+
acceptedAfterBaseline[resource],
|
|
3158
|
+
);
|
|
3159
|
+
}
|
|
3160
|
+
scheduleGovernorState(state, at);
|
|
3161
|
+
return { value: {
|
|
3162
|
+
status: "completed",
|
|
3163
|
+
actualCosts: reservation.actualCosts,
|
|
3164
|
+
accountedCosts: reservation.accountedCosts,
|
|
3165
|
+
} };
|
|
3166
|
+
});
|
|
3167
|
+
}
|
|
3168
|
+
|
|
2521
3169
|
function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
2522
3170
|
return mutateGovernor(scope, undefined, (state, nowMs) => {
|
|
2523
3171
|
const budget = state.budgets[resource];
|
|
@@ -2525,6 +3173,17 @@ function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
|
2525
3173
|
!GOVERNOR_BLOCK_REASONS.has(reason)) return { ok: false, reason: "corrupt" };
|
|
2526
3174
|
budget.blockUntil = resetMs;
|
|
2527
3175
|
budget.blockReason = reason;
|
|
3176
|
+
if (resource === "core") {
|
|
3177
|
+
state.observers.core.nextAt = Math.min(
|
|
3178
|
+
state.observers.core.nextAt || resetMs + BUDGET_RESET_GRACE_MS,
|
|
3179
|
+
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3180
|
+
);
|
|
3181
|
+
} else {
|
|
3182
|
+
state.probeOutcome.nextAt = Math.min(
|
|
3183
|
+
state.probeOutcome.nextAt || resetMs + BUDGET_RESET_GRACE_MS,
|
|
3184
|
+
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3185
|
+
);
|
|
3186
|
+
}
|
|
2528
3187
|
return { value: { resource, resetMs, reason } };
|
|
2529
3188
|
});
|
|
2530
3189
|
}
|
|
@@ -2558,7 +3217,8 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
2558
3217
|
const state = result.value;
|
|
2559
3218
|
let status = "healthy";
|
|
2560
3219
|
if (state.probeClaim?.leaseUntil > nowMs) status = "waiting for probe";
|
|
2561
|
-
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] ||
|
|
3220
|
+
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] ||
|
|
3221
|
+
nowMs - state.budgets[resource].observedAt > budgetSnapshotTtl(resource))) status = "stale";
|
|
2562
3222
|
else if (RATE_RESOURCES.some((resource) => state.budgets[resource].blockUntil > nowMs)) status = "blocked";
|
|
2563
3223
|
return {
|
|
2564
3224
|
status,
|
|
@@ -2569,6 +3229,7 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
2569
3229
|
reserve: resourceReserve(budget.limit),
|
|
2570
3230
|
remaining: budget.remaining,
|
|
2571
3231
|
resetMs: budget.resetMs,
|
|
3232
|
+
source: budget.source,
|
|
2572
3233
|
}]] : [];
|
|
2573
3234
|
})),
|
|
2574
3235
|
};
|
|
@@ -2584,7 +3245,7 @@ async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
|
2584
3245
|
}
|
|
2585
3246
|
|
|
2586
3247
|
async function refreshSharedBudget(scope, leaseId, signal, {
|
|
2587
|
-
readBudgets =
|
|
3248
|
+
readBudgets = readSharedBudgetSources,
|
|
2588
3249
|
now = () => scopeNow(scope),
|
|
2589
3250
|
wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
2590
3251
|
inspect = inspectGovernor,
|
|
@@ -2608,7 +3269,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2608
3269
|
if (
|
|
2609
3270
|
RATE_RESOURCES.every((resource) =>
|
|
2610
3271
|
snapshot.value.budgets[resource] &&
|
|
2611
|
-
now() - snapshot.value.budgets[resource].observedAt <=
|
|
3272
|
+
now() - snapshot.value.budgets[resource].observedAt <= budgetSnapshotTtl(resource),
|
|
2612
3273
|
) && snapshot.value.probeOutcome.status === "healthy"
|
|
2613
3274
|
) {
|
|
2614
3275
|
return { ok: true, value: { status: "published", budgets: snapshot.value.budgets } };
|
|
@@ -2619,7 +3280,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2619
3280
|
claim = claimProbe(scope, leaseId, now());
|
|
2620
3281
|
if (!claim.ok || claim.value.status !== "claimed") return claim;
|
|
2621
3282
|
}
|
|
2622
|
-
const { nonce, startedReservationIds } = claim.value;
|
|
3283
|
+
const { nonce, startedReservationIds, resources = RATE_RESOURCES, coreEtag = null } = claim.value;
|
|
2623
3284
|
const drainUntil = now() + GOVERNOR_PROBE_DRAIN_MS;
|
|
2624
3285
|
while (startedReservationIds.length > 0 && now() < drainUntil) {
|
|
2625
3286
|
const snapshot = inspect(scope, now());
|
|
@@ -2650,7 +3311,24 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2650
3311
|
}
|
|
2651
3312
|
let budgets;
|
|
2652
3313
|
try {
|
|
2653
|
-
budgets = await readBudgets(signal, scope.host
|
|
3314
|
+
budgets = await readBudgets(signal, scope.host, {
|
|
3315
|
+
resources,
|
|
3316
|
+
coreEtag,
|
|
3317
|
+
renewClaim: async () => {
|
|
3318
|
+
const startedAt = now();
|
|
3319
|
+
const extended = await retryGovernorMutation(
|
|
3320
|
+
() => renew(scope, leaseId, nonce, now()),
|
|
3321
|
+
{
|
|
3322
|
+
now,
|
|
3323
|
+
wait,
|
|
3324
|
+
deadline: Math.min(renewed.value.leaseUntil, startedAt + GOVERNOR_PROBE_TRANSITION_MS),
|
|
3325
|
+
signal,
|
|
3326
|
+
},
|
|
3327
|
+
);
|
|
3328
|
+
if (extended.ok) renewed.value.leaseUntil = extended.value.leaseUntil;
|
|
3329
|
+
return extended.ok;
|
|
3330
|
+
},
|
|
3331
|
+
});
|
|
2654
3332
|
} catch {
|
|
2655
3333
|
budgets = null;
|
|
2656
3334
|
}
|
|
@@ -2703,7 +3381,7 @@ function resourcePerTick(table, activeKey) {
|
|
|
2703
3381
|
}
|
|
2704
3382
|
|
|
2705
3383
|
function alertArgs(source, path = source.path) {
|
|
2706
|
-
return [
|
|
3384
|
+
return [apiPath(path), ...apiHostArgs(), "--jq", source.jq];
|
|
2707
3385
|
}
|
|
2708
3386
|
|
|
2709
3387
|
function alertRequestArgs(source) {
|
|
@@ -2778,7 +3456,11 @@ function clearForcedBackoffAfterStart(key, force, status, clear = clearBackoff)
|
|
|
2778
3456
|
return true;
|
|
2779
3457
|
}
|
|
2780
3458
|
|
|
2781
|
-
async function fetchAlertSource(source, signal, now
|
|
3459
|
+
async function fetchAlertSource(source, signal, now, {
|
|
3460
|
+
entities,
|
|
3461
|
+
force = false,
|
|
3462
|
+
request = fetchConditionalEntity,
|
|
3463
|
+
} = {}) {
|
|
2782
3464
|
if (backoffActive(source.key, now)) {
|
|
2783
3465
|
const { note, verdict } = alertBackoff.get(source.key);
|
|
2784
3466
|
// The verdict is replayed alongside the note. Replaying only the note left
|
|
@@ -2790,27 +3472,51 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2790
3472
|
raw: `backoff:${note}`,
|
|
2791
3473
|
completedCalls: 0,
|
|
2792
3474
|
verdict,
|
|
3475
|
+
allNotModified: false,
|
|
3476
|
+
stagedEntities: new Map(),
|
|
2793
3477
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
2794
3478
|
};
|
|
2795
3479
|
}
|
|
2796
3480
|
let completedCalls = 0;
|
|
3481
|
+
const observations = [];
|
|
2797
3482
|
try {
|
|
2798
3483
|
const requests = alertRequestArgs(source);
|
|
2799
3484
|
const payloads = [];
|
|
2800
3485
|
const groups = [];
|
|
3486
|
+
const responses = [];
|
|
2801
3487
|
for (const [index, args] of requests.entries()) {
|
|
2802
3488
|
if (index > 0 && !shouldFetchAlertPriorityLanes(groups[0]?.length ?? 0)) break;
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
3489
|
+
let response = null;
|
|
3490
|
+
try {
|
|
3491
|
+
response = await request({
|
|
3492
|
+
tab: "security",
|
|
3493
|
+
args,
|
|
3494
|
+
operation: "tab:security-endpoint",
|
|
3495
|
+
signal,
|
|
3496
|
+
force,
|
|
3497
|
+
entities,
|
|
3498
|
+
});
|
|
3499
|
+
if (response.status === 200) completedCalls += 1;
|
|
3500
|
+
observations.push(...(response.observations ?? []));
|
|
3501
|
+
responses.push(response);
|
|
3502
|
+
payloads.push(response.body ?? "");
|
|
3503
|
+
groups.push(parseJsonOutput(response.body ?? "").filter((alert) => alert.state === "open"));
|
|
3504
|
+
} catch (error) {
|
|
3505
|
+
if (response === null) completedCalls += 1;
|
|
3506
|
+
observations.push(...(error.budgetObservations ?? []));
|
|
3507
|
+
throw error;
|
|
3508
|
+
}
|
|
2807
3509
|
}
|
|
2808
3510
|
const raw = payloads.join("\0");
|
|
2809
3511
|
clearBackoff(source.key);
|
|
3512
|
+
const batch = conditionalBatchResult(responses, raw, raw);
|
|
2810
3513
|
return {
|
|
2811
3514
|
raw,
|
|
2812
3515
|
completedCalls,
|
|
2813
3516
|
verdict: "ok",
|
|
3517
|
+
allNotModified: batch.allNotModified,
|
|
3518
|
+
stagedEntities: batch.stagedEntities,
|
|
3519
|
+
observations,
|
|
2814
3520
|
parse: () => {
|
|
2815
3521
|
const rows = mergeAlertRows(groups);
|
|
2816
3522
|
return {
|
|
@@ -2827,6 +3533,23 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2827
3533
|
// source's fixed note. Everything else -- an expired SAML session, a rate
|
|
2828
3534
|
// limit, a network drop -- surfaces as itself.
|
|
2829
3535
|
const verdict = classify(err);
|
|
3536
|
+
if (verdict === "unusable-output") {
|
|
3537
|
+
return {
|
|
3538
|
+
raw: "unusable-output",
|
|
3539
|
+
completedCalls,
|
|
3540
|
+
verdict,
|
|
3541
|
+
allNotModified: false,
|
|
3542
|
+
stagedEntities: new Map(),
|
|
3543
|
+
observations,
|
|
3544
|
+
parse: () => ({
|
|
3545
|
+
alerts: [],
|
|
3546
|
+
note: null,
|
|
3547
|
+
verdict,
|
|
3548
|
+
truncated: false,
|
|
3549
|
+
unusable: true,
|
|
3550
|
+
}),
|
|
3551
|
+
};
|
|
3552
|
+
}
|
|
2830
3553
|
const note = verdict === "unavailable" ? source.unavailable : `${source.name}: ${shortErr(err)}`;
|
|
2831
3554
|
const steps = pick(FAILURE_LADDER, verdict, null);
|
|
2832
3555
|
if (steps) {
|
|
@@ -2839,6 +3562,9 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2839
3562
|
// whether it returned data, `[]`, or a 403.
|
|
2840
3563
|
completedCalls,
|
|
2841
3564
|
verdict,
|
|
3565
|
+
allNotModified: false,
|
|
3566
|
+
stagedEntities: new Map(),
|
|
3567
|
+
observations,
|
|
2842
3568
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
2843
3569
|
};
|
|
2844
3570
|
}
|
|
@@ -2856,7 +3582,11 @@ function severityRank(severity) {
|
|
|
2856
3582
|
return Object.hasOwn(SEVERITY_RANK, severity) ? SEVERITY_RANK[severity] : SEVERITY_RANK.unknown;
|
|
2857
3583
|
}
|
|
2858
3584
|
|
|
2859
|
-
async function fetchSecurity(signal
|
|
3585
|
+
async function fetchSecurity(signal, {
|
|
3586
|
+
entities = new Map(),
|
|
3587
|
+
force = false,
|
|
3588
|
+
previousRaw = null,
|
|
3589
|
+
} = {}) {
|
|
2860
3590
|
// Monotonic, not wall-clock. These deadlines measure *elapsed* time, and
|
|
2861
3591
|
// Date.now() can jump: a laptop resume or an NTP correction stepping the clock
|
|
2862
3592
|
// backwards would hold an alert source in backoff for up to an hour of apparent
|
|
@@ -2865,12 +3595,15 @@ async function fetchSecurity(signal) {
|
|
|
2865
3595
|
// for the mirror-image reason: a sleep gap is the thing it reports, and
|
|
2866
3596
|
// performance.now() does not advance across suspend.
|
|
2867
3597
|
const now = performance.now();
|
|
2868
|
-
const parts = await Promise.all(ALERT_SOURCES.map((
|
|
3598
|
+
const parts = await Promise.all(ALERT_SOURCES.map((source) =>
|
|
3599
|
+
fetchAlertSource(source, signal, now, { entities, force })));
|
|
3600
|
+
const allNotModified = parts.length > 0 && parts.every((part) => part.allNotModified);
|
|
3601
|
+
const stagedEntities = new Map(parts.flatMap((part) => [...part.stagedEntities]));
|
|
2869
3602
|
return {
|
|
2870
3603
|
// Joined with a NUL so a change in any one of the three shows up as a
|
|
2871
3604
|
// change in the combined payload, without risk of two different splits
|
|
2872
3605
|
// colliding on the same string.
|
|
2873
|
-
raw: parts.map((p) => p.raw).join("\0"),
|
|
3606
|
+
raw: allNotModified ? previousRaw : parts.map((p) => p.raw).join("\0"),
|
|
2874
3607
|
limit: ALERT_PER_PAGE,
|
|
2875
3608
|
// Summed from what each source reported rather than assumed to be
|
|
2876
3609
|
// ALERT_SOURCES.length: a source held off by backoff spawned nothing and
|
|
@@ -2879,7 +3612,10 @@ async function fetchSecurity(signal) {
|
|
|
2879
3612
|
graphqlSpent: 0,
|
|
2880
3613
|
measuredSuccess: parts.every((part) => part.verdict === "ok"),
|
|
2881
3614
|
rateLimited: parts.some((part) => part.verdict === "rate-limited"),
|
|
3615
|
+
stagedEntities,
|
|
3616
|
+
observations: parts.flatMap((part) => part.observations ?? []),
|
|
2882
3617
|
parse: () => {
|
|
3618
|
+
if (parts.some((part) => part.verdict === "unusable-output")) return { unusable: true };
|
|
2883
3619
|
const parsed = parts.map((p) => p.parse());
|
|
2884
3620
|
const alerts = parsed.flatMap((p) => p.alerts);
|
|
2885
3621
|
alerts.sort((a, b) => {
|
|
@@ -2890,6 +3626,7 @@ async function fetchSecurity(signal) {
|
|
|
2890
3626
|
return {
|
|
2891
3627
|
alerts,
|
|
2892
3628
|
notes: parsed.map((p) => p.note).filter(Boolean),
|
|
3629
|
+
unusable: parsed.some((p) => p.unusable),
|
|
2893
3630
|
// "Blind" is not the same as "switched off", and the tab bar could not
|
|
2894
3631
|
// tell them apart: fetchAlertSource never rejects, so a repo whose three
|
|
2895
3632
|
// endpoints all 403'd on an expired SAML session rendered
|
|
@@ -3267,19 +4004,55 @@ function normalizeRateBudgets(resources) {
|
|
|
3267
4004
|
]));
|
|
3268
4005
|
}
|
|
3269
4006
|
|
|
3270
|
-
async function
|
|
4007
|
+
async function readCoreBudget(signal, host = effectiveRuntimeHost(), etag = null) {
|
|
3271
4008
|
try {
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
4009
|
+
const response = await ghApi(["user", ...apiHostArgs(host)], {
|
|
4010
|
+
signal,
|
|
4011
|
+
operation: "budget-core-observer",
|
|
4012
|
+
etag,
|
|
4013
|
+
});
|
|
4014
|
+
if (response.rateLimit?.resource !== "core") return null;
|
|
4015
|
+
return {
|
|
4016
|
+
budget: response.rateLimit,
|
|
4017
|
+
etag: response.etag ?? etag,
|
|
4018
|
+
receivedAt: Date.now(),
|
|
4019
|
+
};
|
|
4020
|
+
} catch (error) {
|
|
4021
|
+
const status = error?.apiResponse?.status;
|
|
4022
|
+
const rateLimit = pickRateLimit(error?.apiResponse?.headers);
|
|
4023
|
+
if (![403, 429].includes(status) || rateLimit?.resource !== "core") return null;
|
|
4024
|
+
return {
|
|
4025
|
+
budget: rateLimit,
|
|
4026
|
+
etag: error.apiResponse.headers.etag ?? etag,
|
|
4027
|
+
receivedAt: Date.now(),
|
|
4028
|
+
blocked: true,
|
|
4029
|
+
};
|
|
4030
|
+
}
|
|
4031
|
+
}
|
|
4032
|
+
|
|
4033
|
+
async function readSharedBudgetSources(signal, host = effectiveRuntimeHost(), {
|
|
4034
|
+
resources = RATE_RESOURCES,
|
|
4035
|
+
coreEtag = null,
|
|
4036
|
+
renewClaim = null,
|
|
4037
|
+
readGraphql = readRateLimitResources,
|
|
4038
|
+
readCore = readCoreBudget,
|
|
4039
|
+
} = {}) {
|
|
4040
|
+
const result = {};
|
|
4041
|
+
if (resources.includes("graphql")) {
|
|
4042
|
+
const rateResources = await readGraphql(signal, host).catch(() => null);
|
|
4043
|
+
const graphql = normalizeRateBudget(rateResources?.graphql);
|
|
4044
|
+
if (!graphql) return null;
|
|
4045
|
+
result.graphql = { budget: graphql, receivedAt: Date.now(), cost: 0 };
|
|
3282
4046
|
}
|
|
4047
|
+
if (resources.includes("core")) {
|
|
4048
|
+
// Each gh command is independently bounded, but two sequential commands
|
|
4049
|
+
// can outlive one claim. Extend the same nonce after the free GraphQL read
|
|
4050
|
+
// and before the one-cost bootstrap observer can start.
|
|
4051
|
+
if (resources.includes("graphql") && typeof renewClaim === "function" && !await renewClaim()) return null;
|
|
4052
|
+
const core = await readCore(signal, host, coreEtag);
|
|
4053
|
+
if (core) result.core = core;
|
|
4054
|
+
}
|
|
4055
|
+
return Object.keys(result).length > 0 ? result : null;
|
|
3283
4056
|
}
|
|
3284
4057
|
|
|
3285
4058
|
// Conservative unpaced demand at this configuration's floor, derived from the
|
|
@@ -3298,13 +4071,14 @@ function projectedHourlyCost(activeKey) {
|
|
|
3298
4071
|
function doctorProbePlan() {
|
|
3299
4072
|
return [
|
|
3300
4073
|
["Repository access", repoContextArgs(), "doctor:repository"],
|
|
3301
|
-
["Actions
|
|
4074
|
+
["Actions runs", ghApiArgs(actionsRunsArgs(MIN_RUN_LIMIT)), "doctor:actions-runs"],
|
|
4075
|
+
["Actions workflows", ghApiArgs(actionsWorkflowsArgs()), "doctor:actions-workflows"],
|
|
3302
4076
|
["Issues (issue list)", issuesArgs(), "doctor:issues"],
|
|
3303
4077
|
["Pull requests (pr list)", prsArgs(), "doctor:prs"],
|
|
3304
4078
|
...ALERT_SOURCES.flatMap((source) =>
|
|
3305
4079
|
alertRequestArgs(source).map((args, index) => [
|
|
3306
4080
|
index === 0 ? source.name : `${source.name} (priority ${index})`,
|
|
3307
|
-
args,
|
|
4081
|
+
ghApiArgs(args),
|
|
3308
4082
|
"doctor:security-endpoint",
|
|
3309
4083
|
]),
|
|
3310
4084
|
),
|
|
@@ -3349,8 +4123,8 @@ async function runDoctor() {
|
|
|
3349
4123
|
|
|
3350
4124
|
const probes = doctorProbePlan();
|
|
3351
4125
|
|
|
3352
|
-
// Free checks run first.
|
|
3353
|
-
//
|
|
4126
|
+
// Free checks run first. The display rate_limit read is report data only;
|
|
4127
|
+
// admission re-reads its GraphQL source after owning the shared claim.
|
|
3354
4128
|
const [ghVersion, authStatus, remote, remoteUrls] = await Promise.all([
|
|
3355
4129
|
captureGh(["--version"], "version"),
|
|
3356
4130
|
captureGh(["auth", "status"], "auth-status"),
|
|
@@ -3378,13 +4152,7 @@ async function runDoctor() {
|
|
|
3378
4152
|
demand: { core: 6, graphql: 2 },
|
|
3379
4153
|
});
|
|
3380
4154
|
try {
|
|
3381
|
-
|
|
3382
|
-
if (registered.ok && normalized?.core && normalized?.graphql) {
|
|
3383
|
-
const claim = claimProbe(scope, leaseId, nowMs);
|
|
3384
|
-
if (claim.value?.status === "claimed") {
|
|
3385
|
-
publishProbe(scope, leaseId, claim.value.nonce, normalized, nowMs);
|
|
3386
|
-
}
|
|
3387
|
-
}
|
|
4155
|
+
if (registered.ok) await refreshSharedBudget(scope, leaseId, undefined);
|
|
3388
4156
|
const admissionAt = Date.now();
|
|
3389
4157
|
const admittedProbes = probes.map(([name, args, operation], index) => {
|
|
3390
4158
|
const admitted = registered.ok
|
|
@@ -3460,7 +4228,7 @@ async function runDoctor() {
|
|
|
3460
4228
|
return field(
|
|
3461
4229
|
resource,
|
|
3462
4230
|
detail
|
|
3463
|
-
? `${detail.remaining} remaining, ${detail.reserve} reserved, reset ${new Date(detail.resetMs).toISOString()}`
|
|
4231
|
+
? `${detail.remaining} remaining, ${detail.reserve} reserved, reset ${new Date(detail.resetMs).toISOString()} (${detail.source})`
|
|
3464
4232
|
: "unavailable",
|
|
3465
4233
|
);
|
|
3466
4234
|
}),
|
|
@@ -3673,6 +4441,14 @@ function helpLines(maxRows) {
|
|
|
3673
4441
|
const all = keyTableLines();
|
|
3674
4442
|
if (all.length <= rows) return all;
|
|
3675
4443
|
if (rows === 1) return [`… ${all.length} keys: gh-glance --help`];
|
|
4444
|
+
if (rows === 4) {
|
|
4445
|
+
return [
|
|
4446
|
+
all[9],
|
|
4447
|
+
all[6],
|
|
4448
|
+
"Ent/jk Open the selected/Move the cursor",
|
|
4449
|
+
`… ${all.length - 4} more: gh-glance --help`,
|
|
4450
|
+
];
|
|
4451
|
+
}
|
|
3676
4452
|
const priority = [9, 6, 5, 3, 0, 7, 8, 1, 2, 4]
|
|
3677
4453
|
.slice(0, rows - 1)
|
|
3678
4454
|
.map((index) => all[index]);
|
|
@@ -3721,8 +4497,8 @@ from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
|
|
|
3721
4497
|
|
|
3722
4498
|
On a healthy single pane, the active tab is considered every ${REFRESH_MS / 1000}s and each
|
|
3723
4499
|
background tab about every ${(REFRESH_MS * BACKGROUND_EVERY) / 1000}s. Quota-consuming calls still
|
|
3724
|
-
need a shared, resource-specific grant that preserves a hard reserve.
|
|
3725
|
-
and manual refresh do not bypass that safety check.
|
|
4500
|
+
need a shared, resource-specific grant that preserves a hard reserve. Scheduled
|
|
4501
|
+
checks and manual refresh do not bypass that safety check.
|
|
3726
4502
|
|
|
3727
4503
|
Row icons are GitHub Octicons and need a Nerd Font. Without one, set
|
|
3728
4504
|
GH_GLANCE_ICONS=unicode for Unicode status glyphs and text row substitutes, or
|
|
@@ -4520,7 +5296,7 @@ const MemoSecurityRow = React.memo(SecurityRow);
|
|
|
4520
5296
|
const TABS = [
|
|
4521
5297
|
{
|
|
4522
5298
|
key: "actions",
|
|
4523
|
-
fetch: ({ signal, runLimit }) => fetchActions(runLimit, signal),
|
|
5299
|
+
fetch: ({ signal, runLimit, ...options }) => fetchActions(runLimit, signal, options),
|
|
4524
5300
|
label: "Actions",
|
|
4525
5301
|
short: "Actions",
|
|
4526
5302
|
header: ACTIONS_HEADER,
|
|
@@ -4550,7 +5326,7 @@ const TABS = [
|
|
|
4550
5326
|
},
|
|
4551
5327
|
{
|
|
4552
5328
|
key: "security",
|
|
4553
|
-
fetch: ({ signal }) => fetchSecurity(signal),
|
|
5329
|
+
fetch: ({ signal, ...options }) => fetchSecurity(signal, options),
|
|
4554
5330
|
label: "Security",
|
|
4555
5331
|
short: "Security",
|
|
4556
5332
|
header: SECURITY_HEADER,
|
|
@@ -4772,7 +5548,6 @@ function isRecord(value) {
|
|
|
4772
5548
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4773
5549
|
}
|
|
4774
5550
|
|
|
4775
|
-
const persistenceWaitCell = new Int32Array(new SharedArrayBuffer(4));
|
|
4776
5551
|
const PERSISTENCE_LOCK_WAIT_MS = 250;
|
|
4777
5552
|
const PERSISTENCE_STALE_LOCK_MS = 5000;
|
|
4778
5553
|
|
|
@@ -5501,16 +6276,42 @@ function governorWakeTimes(state, nowMs, floorMs, leaseId = null) {
|
|
|
5501
6276
|
};
|
|
5502
6277
|
}
|
|
5503
6278
|
|
|
5504
|
-
function
|
|
6279
|
+
function governorProtocolReady(refreshResult, snapshot, nowMs) {
|
|
5505
6280
|
if (!refreshResult?.ok || !snapshot?.ok) return false;
|
|
5506
6281
|
if (["waiting", "paused", "probe"].includes(refreshResult.value?.status)) return false;
|
|
5507
6282
|
if (snapshot.value.probeClaim || snapshot.value.probeOutcome?.status !== "healthy") return false;
|
|
6283
|
+
return RATE_RESOURCES.every((resource) => {
|
|
6284
|
+
const budget = snapshot.value.budgets[resource];
|
|
6285
|
+
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource);
|
|
6286
|
+
});
|
|
6287
|
+
}
|
|
6288
|
+
|
|
6289
|
+
function governorControlReady(refreshResult, snapshot, nowMs) {
|
|
6290
|
+
if (governorProtocolReady(refreshResult, snapshot, nowMs)) return true;
|
|
6291
|
+
// A losing probe claimant can exhaust its bounded reinspection just before
|
|
6292
|
+
// the winner publishes. Its return value is still `waiting`, but the locked
|
|
6293
|
+
// snapshot taken immediately afterwards is authoritative. Adopt only that
|
|
6294
|
+
// successful handoff; failures and every incomplete snapshot remain closed.
|
|
6295
|
+
return refreshResult?.ok && refreshResult.value?.status === "waiting" &&
|
|
6296
|
+
governorProtocolReady({ ok: true, value: { status: "published" } }, snapshot, nowMs);
|
|
6297
|
+
}
|
|
6298
|
+
|
|
6299
|
+
function tabEpochChanged(previous, next, key) {
|
|
6300
|
+
const costs = tabRequestCost(key);
|
|
6301
|
+
if (!costs || !isRecord(previous) || !isRecord(next)) return false;
|
|
6302
|
+
return RATE_RESOURCES.some((resource) =>
|
|
6303
|
+
costs[resource] > 0 && previous[resource] !== next[resource]);
|
|
6304
|
+
}
|
|
6305
|
+
|
|
6306
|
+
function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
|
|
6307
|
+
if (!governorProtocolReady(refreshResult, snapshot, nowMs)) return false;
|
|
5508
6308
|
const costs = tabRequestCost(activeKey);
|
|
5509
6309
|
return RATE_RESOURCES.every((resource) => {
|
|
5510
6310
|
if (costs[resource] <= 0) return true;
|
|
5511
6311
|
const budget = snapshot.value.budgets[resource];
|
|
5512
|
-
return budget && nowMs - budget.observedAt <=
|
|
5513
|
-
budget.blockUntil <= nowMs &&
|
|
6312
|
+
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
|
|
6313
|
+
budget.blockUntil <= nowMs &&
|
|
6314
|
+
availableForGrant({ budget, resource, nowMs }).mode === "open";
|
|
5514
6315
|
});
|
|
5515
6316
|
}
|
|
5516
6317
|
|
|
@@ -5582,8 +6383,8 @@ function pendingFailureIsTerminal(reason) {
|
|
|
5582
6383
|
return reason === "stale";
|
|
5583
6384
|
}
|
|
5584
6385
|
|
|
5585
|
-
function runtimeIntentGate(liveScheduling, { force = false } = {}) {
|
|
5586
|
-
return liveScheduling
|
|
6386
|
+
function runtimeIntentGate(liveScheduling, { force = false, protocolReady = false } = {}) {
|
|
6387
|
+
return liveScheduling || protocolReady
|
|
5587
6388
|
? { registerIntent: true, requestProbe: false }
|
|
5588
6389
|
: { registerIntent: false, requestProbe: force === true };
|
|
5589
6390
|
}
|
|
@@ -5635,6 +6436,13 @@ function rateLimitBlockDecision(results, resetMs) {
|
|
|
5635
6436
|
};
|
|
5636
6437
|
}
|
|
5637
6438
|
|
|
6439
|
+
function coordinationNotice(reason) {
|
|
6440
|
+
if (reason === "unknown-scope") return "Confirming your GitHub login…";
|
|
6441
|
+
if (reason === "block-unpublished") return "Holding until the rate-limit block is shared";
|
|
6442
|
+
if (reason === "busy" || reason === "stale") return "Coordinating with your other panes";
|
|
6443
|
+
return "Can't coordinate API use — retrying";
|
|
6444
|
+
}
|
|
6445
|
+
|
|
5638
6446
|
function retryRateLimitBlockPublication(pending, nowMs, publish) {
|
|
5639
6447
|
const blocks = Array.isArray(pending?.blocks) ? pending.blocks : [];
|
|
5640
6448
|
if (blocks.length === 0 || blocks.some((block) =>
|
|
@@ -5728,7 +6536,14 @@ async function runAdmittedOperation({ scope, leaseId, operation, priority = "man
|
|
|
5728
6536
|
|
|
5729
6537
|
function pollResultTransition({ key, previousRaw, raw, parse, limit, completedAt }) {
|
|
5730
6538
|
if (previousRaw === raw) return { kind: "unchanged", completedAt };
|
|
5731
|
-
|
|
6539
|
+
let value;
|
|
6540
|
+
try {
|
|
6541
|
+
value = parse();
|
|
6542
|
+
} catch (error) {
|
|
6543
|
+
if (isUnusableOutput(error)) return { kind: "unusable", nextRaw: null };
|
|
6544
|
+
throw error;
|
|
6545
|
+
}
|
|
6546
|
+
if (value?.unusable) return { kind: "unusable", nextRaw: null };
|
|
5732
6547
|
if (key === "security" && value?.blind) {
|
|
5733
6548
|
return {
|
|
5734
6549
|
kind: "blind",
|
|
@@ -5890,13 +6705,17 @@ const REMOTE_SETUP_NONINTERACTIVE_LINES = [
|
|
|
5890
6705
|
// Reserved so the hints never shift when the active tab changes state. Every
|
|
5891
6706
|
// status label below fits this fixed cell in both icon profiles.
|
|
5892
6707
|
const REFRESH_STATUS_WIDTH = 12;
|
|
6708
|
+
const WIDE_STATE_WIDTH = REFRESH_STATUS_WIDTH + 1 + 12;
|
|
6709
|
+
// StatusBar receives the drawable frame width, which is one column narrower
|
|
6710
|
+
// than the terminal. A 44-cell threshold preserves the 45-column contract.
|
|
6711
|
+
const WIDE_STATE_MIN_COLS = 44;
|
|
6712
|
+
const NOTICE_ROWS = 1;
|
|
5893
6713
|
|
|
5894
6714
|
const REFRESH_STATUS_GLYPHS = Object.freeze({
|
|
5895
6715
|
unicode: Object.freeze({
|
|
5896
6716
|
setup: "·",
|
|
5897
6717
|
checking: SPINNER[0],
|
|
5898
6718
|
paused: "‖",
|
|
5899
|
-
waiting: "·",
|
|
5900
6719
|
failed: "!",
|
|
5901
6720
|
limited: "?",
|
|
5902
6721
|
watching: "·",
|
|
@@ -5905,7 +6724,6 @@ const REFRESH_STATUS_GLYPHS = Object.freeze({
|
|
|
5905
6724
|
setup: ".",
|
|
5906
6725
|
checking: ".",
|
|
5907
6726
|
paused: "|",
|
|
5908
|
-
waiting: ".",
|
|
5909
6727
|
failed: "!",
|
|
5910
6728
|
limited: "?",
|
|
5911
6729
|
watching: ".",
|
|
@@ -5954,30 +6772,38 @@ function refreshStatus({
|
|
|
5954
6772
|
return status("paused", "paused", "Paused", "attention", false, detailKind);
|
|
5955
6773
|
}
|
|
5956
6774
|
if (["waiting", "pending", "probe"].includes(mode)) {
|
|
5957
|
-
const
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
6775
|
+
const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
|
|
6776
|
+
const watchingDetail = sharing
|
|
6777
|
+
? "sharing"
|
|
6778
|
+
: Number.isFinite(governorDecision?.notBefore) ? "next" : detailKind;
|
|
6779
|
+
return status("watching", "watching", "Watching", "inert", false, watchingDetail);
|
|
5961
6780
|
}
|
|
5962
6781
|
if (activeError) return status("failed", "failed", "Failed", "attention");
|
|
5963
6782
|
if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
|
|
5964
6783
|
return status("watching", "watching", "Watching", "inert");
|
|
5965
6784
|
}
|
|
5966
6785
|
|
|
5967
|
-
function
|
|
5968
|
-
if (!Number.isFinite(at)) return null;
|
|
5969
|
-
const
|
|
5970
|
-
if (
|
|
5971
|
-
|
|
6786
|
+
function statusInterval(at, nowMs = Date.now()) {
|
|
6787
|
+
if (!Number.isFinite(at) || !Number.isFinite(nowMs)) return null;
|
|
6788
|
+
const remaining = at - nowMs;
|
|
6789
|
+
if (remaining < 60_000) return "<1m";
|
|
6790
|
+
const minutes = Math.ceil(remaining / 60_000);
|
|
6791
|
+
return minutes > 99 ? "99m+" : `${minutes}m`;
|
|
5972
6792
|
}
|
|
5973
6793
|
|
|
5974
|
-
function statusDetailVariants(status, detail) {
|
|
6794
|
+
function statusDetailVariants(status, detail, nowMs) {
|
|
5975
6795
|
if (!status?.detailKind) return [];
|
|
5976
6796
|
if (status.detailKind === "probing") return ["probing"];
|
|
6797
|
+
if (status.detailKind === "sharing") {
|
|
6798
|
+
const sharing = sharedLaneProvenance(detail);
|
|
6799
|
+
return sharing.waitCause
|
|
6800
|
+
? [`sharing ${sharing.sharingCount}`]
|
|
6801
|
+
: [];
|
|
6802
|
+
}
|
|
5977
6803
|
const at = status.detailKind === "next" ? detail?.notBefore : detail?.resetMs;
|
|
5978
|
-
const
|
|
5979
|
-
if (!
|
|
5980
|
-
return [`${status.detailKind} ${
|
|
6804
|
+
const interval = statusInterval(at, nowMs);
|
|
6805
|
+
if (!interval) return [];
|
|
6806
|
+
return [`${status.detailKind} ${interval}`, interval];
|
|
5981
6807
|
}
|
|
5982
6808
|
|
|
5983
6809
|
function statusBarLayout({
|
|
@@ -5987,20 +6813,25 @@ function statusBarLayout({
|
|
|
5987
6813
|
status,
|
|
5988
6814
|
detail = null,
|
|
5989
6815
|
stale = null,
|
|
6816
|
+
nowMs = Date.now(),
|
|
5990
6817
|
version: currentVersion = "",
|
|
5991
6818
|
} = {}) {
|
|
5992
6819
|
const width = Number.isSafeInteger(cols) ? Math.max(0, cols) : 0;
|
|
5993
6820
|
const hints = Array.isArray(availableHints) ? availableHints : [];
|
|
5994
6821
|
const mandatory = hints.filter(isMandatoryHint);
|
|
5995
6822
|
const optional = hints.filter((hint) => !isMandatoryHint(hint));
|
|
5996
|
-
|
|
6823
|
+
const stateWidth = width >= WIDE_STATE_MIN_COLS
|
|
6824
|
+
? Math.min(width, WIDE_STATE_WIDTH)
|
|
6825
|
+
: Math.min(width, REFRESH_STATUS_WIDTH);
|
|
6826
|
+
let used = stateWidth;
|
|
5997
6827
|
const selectedHints = [];
|
|
5998
6828
|
const separatorWidth = () => selectedHints.length > 0 ? 1 : 0;
|
|
5999
6829
|
const addHint = (hint, showLabel) => {
|
|
6000
6830
|
const text = showLabel ? `${hint.label}: ${hint.keys}` : hint.keys;
|
|
6001
|
-
const
|
|
6831
|
+
const separator = separatorWidth();
|
|
6832
|
+
const cost = separator + [...text].length;
|
|
6002
6833
|
if (used + cost > width) return false;
|
|
6003
|
-
selectedHints.push({ ...hint, showLabel, text });
|
|
6834
|
+
selectedHints.push({ ...hint, showLabel, text, start: used + separator });
|
|
6004
6835
|
used += cost;
|
|
6005
6836
|
return true;
|
|
6006
6837
|
};
|
|
@@ -6020,20 +6851,17 @@ function statusBarLayout({
|
|
|
6020
6851
|
else addHint(hint, false);
|
|
6021
6852
|
}
|
|
6022
6853
|
|
|
6854
|
+
const payloadWidth = stateWidth > REFRESH_STATUS_WIDTH
|
|
6855
|
+
? stateWidth - REFRESH_STATUS_WIDTH - 1
|
|
6856
|
+
: 0;
|
|
6857
|
+
const staleText = typeof stale === "string" ? stale : null;
|
|
6858
|
+
const selectedStale = staleText && [...staleText].length <= payloadWidth ? staleText : null;
|
|
6023
6859
|
let selectedDetail = null;
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
selectedDetail = candidate;
|
|
6028
|
-
used += cost;
|
|
6029
|
-
break;
|
|
6030
|
-
}
|
|
6860
|
+
if (!selectedStale) {
|
|
6861
|
+
selectedDetail = statusDetailVariants(status, detail, nowMs)
|
|
6862
|
+
.find((candidate) => [...candidate].length <= payloadWidth) ?? null;
|
|
6031
6863
|
}
|
|
6032
6864
|
|
|
6033
|
-
const staleText = typeof stale === "string" ? stale : null;
|
|
6034
|
-
const selectedStale = staleText && used + 1 + staleText.length <= width ? staleText : null;
|
|
6035
|
-
if (selectedStale) used += 1 + selectedStale.length;
|
|
6036
|
-
|
|
6037
6865
|
for (const hint of optional) {
|
|
6038
6866
|
if (!addHint(hint, true)) addHint(hint, false);
|
|
6039
6867
|
}
|
|
@@ -6041,7 +6869,7 @@ function statusBarLayout({
|
|
|
6041
6869
|
const versionText = String(currentVersion ?? "");
|
|
6042
6870
|
const showVersion = versionText.length > 0 && used + 1 + versionText.length <= width;
|
|
6043
6871
|
return {
|
|
6044
|
-
|
|
6872
|
+
stateWidth,
|
|
6045
6873
|
hints: selectedHints,
|
|
6046
6874
|
mandatoryHints: selectedHints.filter(isMandatoryHint),
|
|
6047
6875
|
optionalHints: selectedHints.filter((hint) => !isMandatoryHint(hint)),
|
|
@@ -6055,11 +6883,13 @@ function statusBarLayout({
|
|
|
6055
6883
|
function freshnessDeadline({
|
|
6056
6884
|
lastOk,
|
|
6057
6885
|
refreshMs,
|
|
6886
|
+
grantedMs = null,
|
|
6058
6887
|
governorDecision = null,
|
|
6059
6888
|
currentEpochs = null,
|
|
6060
6889
|
} = {}) {
|
|
6061
6890
|
if (!Number.isFinite(lastOk) || !Number.isFinite(refreshMs) || refreshMs < 0) return null;
|
|
6062
|
-
const
|
|
6891
|
+
const cadenceMs = Number.isFinite(grantedMs) && grantedMs >= 0 ? grantedMs : refreshMs;
|
|
6892
|
+
const baseDeadline = lastOk + Math.max(STALE_AFTER_MS, cadenceMs * 6);
|
|
6063
6893
|
const decisionEpochs = governorDecision?.epochs;
|
|
6064
6894
|
const chargedEpochs = isRecord(decisionEpochs)
|
|
6065
6895
|
? Object.entries(decisionEpochs).filter(([, epoch]) => epoch !== null)
|
|
@@ -6075,6 +6905,16 @@ function freshnessDeadline({
|
|
|
6075
6905
|
: baseDeadline;
|
|
6076
6906
|
}
|
|
6077
6907
|
|
|
6908
|
+
function nextAdmittedCadence(previous, startedAt) {
|
|
6909
|
+
if (!Number.isFinite(startedAt)) return previous ?? { startedAt: null, grantedMs: null };
|
|
6910
|
+
return {
|
|
6911
|
+
startedAt,
|
|
6912
|
+
grantedMs: Number.isFinite(previous?.startedAt)
|
|
6913
|
+
? Math.max(0, startedAt - previous.startedAt)
|
|
6914
|
+
: null,
|
|
6915
|
+
};
|
|
6916
|
+
}
|
|
6917
|
+
|
|
6078
6918
|
function probingGovernorDecisions() {
|
|
6079
6919
|
return Object.fromEntries(TABS.map((candidate) => [candidate.key, {
|
|
6080
6920
|
mode: "waiting",
|
|
@@ -6099,6 +6939,8 @@ function sameVisibleGovernorDecision(left, right) {
|
|
|
6099
6939
|
"resource",
|
|
6100
6940
|
"probing",
|
|
6101
6941
|
"detailKind",
|
|
6942
|
+
"waitCause",
|
|
6943
|
+
"sharingCount",
|
|
6102
6944
|
"coordinationError",
|
|
6103
6945
|
"reservationId",
|
|
6104
6946
|
]) {
|
|
@@ -6149,6 +6991,7 @@ function StatusBar({
|
|
|
6149
6991
|
detail,
|
|
6150
6992
|
spin,
|
|
6151
6993
|
stale,
|
|
6994
|
+
nowMs,
|
|
6152
6995
|
interactive,
|
|
6153
6996
|
cols,
|
|
6154
6997
|
remoteSetup = false,
|
|
@@ -6190,6 +7033,7 @@ function StatusBar({
|
|
|
6190
7033
|
status: semanticStatus,
|
|
6191
7034
|
detail,
|
|
6192
7035
|
stale,
|
|
7036
|
+
nowMs,
|
|
6193
7037
|
version,
|
|
6194
7038
|
});
|
|
6195
7039
|
const profile = ICON_PROFILE === "ascii" ? "ascii" : "unicode";
|
|
@@ -6218,20 +7062,20 @@ function StatusBar({
|
|
|
6218
7062
|
{ flexDirection: "row" },
|
|
6219
7063
|
e(
|
|
6220
7064
|
Box,
|
|
6221
|
-
{ width: layout.
|
|
7065
|
+
{ width: layout.stateWidth, flexShrink: 0, flexDirection: "row" },
|
|
6222
7066
|
e(
|
|
6223
7067
|
Text,
|
|
6224
7068
|
{ color: statusColor, dimColor: semanticStatus.tone === "inert", wrap: "truncate-end" },
|
|
6225
7069
|
`${glyph} ${semanticStatus.label}`,
|
|
6226
7070
|
),
|
|
7071
|
+
layout.detail
|
|
7072
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { dimColor: true }, layout.detail))
|
|
7073
|
+
: null,
|
|
7074
|
+
layout.stale
|
|
7075
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, layout.stale))
|
|
7076
|
+
: null,
|
|
6227
7077
|
),
|
|
6228
7078
|
...layout.mandatoryHints.flatMap((hint, index) => renderHint(hint, index, "mandatory")),
|
|
6229
|
-
layout.detail
|
|
6230
|
-
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { dimColor: true }, layout.detail))
|
|
6231
|
-
: null,
|
|
6232
|
-
layout.stale
|
|
6233
|
-
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, layout.stale))
|
|
6234
|
-
: null,
|
|
6235
7079
|
...layout.optionalHints.flatMap((hint, index) => renderHint(hint, index, "optional")),
|
|
6236
7080
|
layout.version && e(Box, { flexGrow: 1 }),
|
|
6237
7081
|
layout.version && e(Text, { key: "version", dimColor: true, wrap: "truncate-end" }, layout.version),
|
|
@@ -6399,6 +7243,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6399
7243
|
const [requestStatuses, setRequestStatuses] = useState(() =>
|
|
6400
7244
|
Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
|
|
6401
7245
|
);
|
|
7246
|
+
const [visibleCoordinationCondition, setVisibleCoordinationCondition] = useState(null);
|
|
6402
7247
|
const [now, setNow] = useState(new Date());
|
|
6403
7248
|
const { rows, cols, useShortLabels } = useTerminalSize(stdout);
|
|
6404
7249
|
const [frame, setFrame] = useState(0);
|
|
@@ -6443,8 +7288,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6443
7288
|
// Counts the lines actually rendered, not the notes collected -- getting this
|
|
6444
7289
|
// wrong by one row is what makes ink repaint the whole frame.
|
|
6445
7290
|
const extraLines =
|
|
6446
|
-
|
|
6447
|
-
(activeGovernorDecision?.coordinationError && !remoteSetup ? 1 : 0) +
|
|
7291
|
+
NOTICE_ROWS +
|
|
6448
7292
|
(tab.key === "security" && !remoteSetup ? securityLines.length : 0);
|
|
6449
7293
|
// Reserve lines for: the tab bar and the divider under it (2), the panel's
|
|
6450
7294
|
// top and bottom edges (2), the column header and its separator (2), and
|
|
@@ -6475,7 +7319,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6475
7319
|
ANIMATE && tab.key === "actions" && (data.actions ?? []).some((r) => r.status === "in_progress");
|
|
6476
7320
|
|
|
6477
7321
|
const inFlightRef = useRef({});
|
|
7322
|
+
const admittedCadenceRef = useRef({});
|
|
6478
7323
|
const rawRef = useRef({});
|
|
7324
|
+
const entityRef = useRef(new Map());
|
|
6479
7325
|
// Last *successful* poll per tab, wall-clock. Wall-clock on purpose: a laptop
|
|
6480
7326
|
// sleeping is exactly the gap this is meant to report, and a monotonic clock
|
|
6481
7327
|
// does not advance across suspend. Never written on the failure path, or a
|
|
@@ -6992,6 +7838,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6992
7838
|
status: "scheduled",
|
|
6993
7839
|
reservationId: `reservation:${pendingIntent.intentId}`,
|
|
6994
7840
|
...reservation,
|
|
7841
|
+
...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
|
|
6995
7842
|
}));
|
|
6996
7843
|
return;
|
|
6997
7844
|
}
|
|
@@ -7027,6 +7874,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7027
7874
|
force = false,
|
|
7028
7875
|
scope,
|
|
7029
7876
|
reservationId,
|
|
7877
|
+
admittedAt,
|
|
7030
7878
|
onSettled,
|
|
7031
7879
|
automaticStatusVisible = false,
|
|
7032
7880
|
} = {}) {
|
|
@@ -7035,6 +7883,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7035
7883
|
// background fetch -- and so a slow repo can't stack refreshes.
|
|
7036
7884
|
if (inFlightRef.current[key]) return Promise.resolve();
|
|
7037
7885
|
inFlightRef.current[key] = true;
|
|
7886
|
+
if (force) manualInFlight.add(key);
|
|
7887
|
+
admittedCadenceRef.current[key] = nextAdmittedCadence(
|
|
7888
|
+
admittedCadenceRef.current[key],
|
|
7889
|
+
admittedAt,
|
|
7890
|
+
);
|
|
7038
7891
|
clearForcedBackoffAfterStart(key, force, "started");
|
|
7039
7892
|
const visibleLoading = shouldShowFetchLoading({
|
|
7040
7893
|
hasData: dataRef.current[key] !== null,
|
|
@@ -7056,14 +7909,15 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7056
7909
|
}
|
|
7057
7910
|
return run()
|
|
7058
7911
|
.then((result) => {
|
|
7059
|
-
|
|
7060
|
-
? { outcome: "rejected" }
|
|
7912
|
+
settleReservationWithBudgetObservations(scope, leaseId, reservationId, result?.measuredSuccess === false
|
|
7913
|
+
? { outcome: "rejected", observations: result?.observations ?? [] }
|
|
7061
7914
|
: {
|
|
7062
7915
|
outcome: "measured-success",
|
|
7063
|
-
|
|
7916
|
+
actualCosts: {
|
|
7064
7917
|
core: result?.restSpent ?? REST_PER_FETCH[key] ?? 0,
|
|
7065
7918
|
graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
|
|
7066
7919
|
},
|
|
7920
|
+
observations: result?.observations ?? [],
|
|
7067
7921
|
}, Date.now());
|
|
7068
7922
|
if (result?.rateLimited) {
|
|
7069
7923
|
const blockAt = Date.now();
|
|
@@ -7075,7 +7929,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7075
7929
|
}], blockAt);
|
|
7076
7930
|
}
|
|
7077
7931
|
if (cancelled) return;
|
|
7078
|
-
const { raw, parse, limit } = result;
|
|
7932
|
+
const { raw, parse, limit, stagedEntities } = result;
|
|
7079
7933
|
// Identical payload: skip the parse *and* the state update. Returning
|
|
7080
7934
|
// the same state object makes React bail out of the re-render, so an
|
|
7081
7935
|
// idle repo stops redrawing the pane entirely.
|
|
@@ -7114,6 +7968,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7114
7968
|
force,
|
|
7115
7969
|
});
|
|
7116
7970
|
}
|
|
7971
|
+
publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
|
|
7117
7972
|
if (transition.kind === "unchanged") {
|
|
7118
7973
|
lastOkRef.current[key] = completedAt;
|
|
7119
7974
|
// Clear on the first success or a single failure latches the ladder.
|
|
@@ -7140,6 +7995,16 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7140
7995
|
}
|
|
7141
7996
|
return;
|
|
7142
7997
|
}
|
|
7998
|
+
// Empty or truncated JSON is not a user-actionable fetch error. Keep
|
|
7999
|
+
// the last-good rows and freshness clock, and clear the raw comparison
|
|
8000
|
+
// so the next admitted tick parses its response instead of taking the
|
|
8001
|
+
// identical-output fast path.
|
|
8002
|
+
if (transition.kind === "unusable") {
|
|
8003
|
+
if (key === "security") securityNextPollAt = 0;
|
|
8004
|
+
rawRef.current[key] = transition.nextRaw;
|
|
8005
|
+
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
8006
|
+
return;
|
|
8007
|
+
}
|
|
7143
8008
|
// Security fetches resolve each source independently so their notes
|
|
7144
8009
|
// remain visible. A blind result is still a failed observation: it
|
|
7145
8010
|
// must not replace known alerts with a false empty state or advance
|
|
@@ -7170,8 +8035,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7170
8035
|
});
|
|
7171
8036
|
})
|
|
7172
8037
|
.catch((err) => {
|
|
7173
|
-
|
|
8038
|
+
settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
|
|
7174
8039
|
outcome: governorOutcomeForError(err),
|
|
8040
|
+
observations: err?.budgetObservations ?? [],
|
|
7175
8041
|
}, Date.now());
|
|
7176
8042
|
if (cancelled || err?.name === "AbortError") return;
|
|
7177
8043
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
@@ -7207,6 +8073,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7207
8073
|
})
|
|
7208
8074
|
.finally(() => {
|
|
7209
8075
|
inFlightRef.current[key] = false;
|
|
8076
|
+
manualInFlight.delete(key);
|
|
7210
8077
|
onSettled?.();
|
|
7211
8078
|
if (!cancelled) {
|
|
7212
8079
|
setLoading((l) => (l[key] ? { ...l, [key]: false } : l));
|
|
@@ -7222,17 +8089,26 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7222
8089
|
: current);
|
|
7223
8090
|
}
|
|
7224
8091
|
}
|
|
8092
|
+
if (!cancelled && queuedManual.delete(key)) {
|
|
8093
|
+
void requestTab(key, "manual", { force: true });
|
|
8094
|
+
}
|
|
7225
8095
|
});
|
|
7226
8096
|
}
|
|
7227
8097
|
|
|
7228
8098
|
const leaseId = governorId();
|
|
7229
8099
|
const pending = new Map();
|
|
8100
|
+
// One manual handoff per tab. A keypress during an automatic request must
|
|
8101
|
+
// not be lost, but repeated keypresses during the forced request itself
|
|
8102
|
+
// must not create a trailing second batch.
|
|
8103
|
+
const queuedManual = new Set();
|
|
8104
|
+
const manualInFlight = new Set();
|
|
7230
8105
|
const wakeScheduler = createWakeScheduler();
|
|
7231
8106
|
let scope = null;
|
|
7232
8107
|
let cleanupScope = null;
|
|
7233
8108
|
let registeredScopeHash = null;
|
|
7234
8109
|
let remoteUrls = [];
|
|
7235
8110
|
let liveScheduling = false;
|
|
8111
|
+
let controlEpochs = null;
|
|
7236
8112
|
let activePollAt = Number.POSITIVE_INFINITY;
|
|
7237
8113
|
let backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7238
8114
|
let backgroundIndex = 0;
|
|
@@ -7244,6 +8120,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7244
8120
|
|
|
7245
8121
|
function failClosedRateLimit(key, reason) {
|
|
7246
8122
|
liveScheduling = false;
|
|
8123
|
+
controlEpochs = null;
|
|
7247
8124
|
activePollAt = Number.POSITIVE_INFINITY;
|
|
7248
8125
|
backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7249
8126
|
wakeScheduler.clear("data");
|
|
@@ -7400,6 +8277,12 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7400
8277
|
armFromState();
|
|
7401
8278
|
}
|
|
7402
8279
|
|
|
8280
|
+
function replaceActivePoll(kind, at) {
|
|
8281
|
+
if (["manual", "tab-switch"].includes(kind)) {
|
|
8282
|
+
activePollAt = at + runtime.refreshMs;
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
|
|
7403
8286
|
function requestTab(key, kind = "active", { force = false } = {}) {
|
|
7404
8287
|
const signal = controller.signal;
|
|
7405
8288
|
const descriptor = tabForKey(key);
|
|
@@ -7408,7 +8291,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7408
8291
|
pauseCoordination(key, "block-unpublished");
|
|
7409
8292
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7410
8293
|
}
|
|
7411
|
-
if (inFlightRef.current[key])
|
|
8294
|
+
if (inFlightRef.current[key]) {
|
|
8295
|
+
if (force && !manualInFlight.has(key)) queuedManual.add(key);
|
|
8296
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
8297
|
+
}
|
|
7412
8298
|
if (!force && backoffActive(`tab:${key}`, monotonicNow)) {
|
|
7413
8299
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7414
8300
|
}
|
|
@@ -7421,20 +8307,39 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7421
8307
|
pauseCoordination(key, "unknown-scope");
|
|
7422
8308
|
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7423
8309
|
}
|
|
7424
|
-
|
|
8310
|
+
let protocolReady = liveScheduling;
|
|
8311
|
+
let resourceReady = liveScheduling;
|
|
8312
|
+
if (!liveScheduling || force) {
|
|
8313
|
+
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
8314
|
+
protocolReady = governorProtocolReady(
|
|
8315
|
+
{ ok: true, value: { status: "published" } },
|
|
8316
|
+
snapshot,
|
|
8317
|
+
nowMs,
|
|
8318
|
+
);
|
|
8319
|
+
resourceReady = governorDataReady(
|
|
8320
|
+
{ ok: true, value: { status: "published" } }, snapshot, key, nowMs);
|
|
8321
|
+
if (!liveScheduling && resourceReady) {
|
|
8322
|
+
liveScheduling = true;
|
|
8323
|
+
activePollAt = nowMs;
|
|
8324
|
+
backgroundPollAt = nowMs + 4 * runtime.refreshMs;
|
|
8325
|
+
armWake("heartbeat", nowMs + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
8326
|
+
}
|
|
8327
|
+
}
|
|
8328
|
+
const intentGate = runtimeIntentGate(liveScheduling, { force, protocolReady });
|
|
8329
|
+
if (intentGate.requestProbe || force && !resourceReady) {
|
|
8330
|
+
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
8331
|
+
const costs = tabRequestCost(key);
|
|
8332
|
+
for (const resource of RATE_RESOURCES) {
|
|
8333
|
+
const budget = snapshot.value?.budgets?.[resource];
|
|
8334
|
+
if (costs[resource] > 0 && budget) {
|
|
8335
|
+
requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
8336
|
+
}
|
|
8337
|
+
}
|
|
8338
|
+
armWake("control", nowMs + 1, controlWake);
|
|
8339
|
+
}
|
|
7425
8340
|
if (!intentGate.registerIntent) {
|
|
7426
8341
|
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
7427
8342
|
setTabWaiting(key, true);
|
|
7428
|
-
if (intentGate.requestProbe) {
|
|
7429
|
-
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
7430
|
-
const costs = tabRequestCost(key);
|
|
7431
|
-
for (const resource of RATE_RESOURCES) {
|
|
7432
|
-
const budget = snapshot.value?.budgets?.[resource];
|
|
7433
|
-
if (costs[resource] > 0 && budget) {
|
|
7434
|
-
requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
7435
|
-
}
|
|
7436
|
-
}
|
|
7437
|
-
}
|
|
7438
8343
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7439
8344
|
}
|
|
7440
8345
|
const existing = pending.get(key);
|
|
@@ -7460,8 +8365,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7460
8365
|
pauseCoordination(key, registered.reason);
|
|
7461
8366
|
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7462
8367
|
}
|
|
7463
|
-
pending.set(key, { intentId, kind, force, wasDeferred: false });
|
|
7464
8368
|
const decision = registered.value;
|
|
8369
|
+
pending.set(key, {
|
|
8370
|
+
intentId,
|
|
8371
|
+
kind,
|
|
8372
|
+
force,
|
|
8373
|
+
wasDeferred: false,
|
|
8374
|
+
...sharedLaneEvidence(decision),
|
|
8375
|
+
});
|
|
7465
8376
|
if (decision.status !== "scheduled" || decision.notBefore > nowMs) {
|
|
7466
8377
|
const item = pending.get(key);
|
|
7467
8378
|
if (item) item.wasDeferred = true;
|
|
@@ -7493,14 +8404,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7493
8404
|
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7494
8405
|
}
|
|
7495
8406
|
setTabWaiting(key, false);
|
|
8407
|
+
replaceActivePoll(kind, nowMs);
|
|
7496
8408
|
if (force) coordinator.invalidate();
|
|
7497
8409
|
return commit(
|
|
7498
8410
|
key,
|
|
7499
|
-
() => descriptor.fetch({
|
|
8411
|
+
() => descriptor.fetch({
|
|
8412
|
+
signal,
|
|
8413
|
+
runLimit: runLimitRef.current,
|
|
8414
|
+
entities: entityRef.current,
|
|
8415
|
+
force,
|
|
8416
|
+
previousRaw: rawRef.current[key] ?? null,
|
|
8417
|
+
}),
|
|
7500
8418
|
{
|
|
7501
8419
|
force,
|
|
7502
8420
|
scope: currentScope,
|
|
7503
8421
|
reservationId: decision.reservationId,
|
|
8422
|
+
admittedAt: nowMs,
|
|
7504
8423
|
automaticStatusVisible: false,
|
|
7505
8424
|
onSettled: () => finishPending(key, intentId),
|
|
7506
8425
|
},
|
|
@@ -7508,7 +8427,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7508
8427
|
}
|
|
7509
8428
|
fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
|
|
7510
8429
|
const requestedAt = Date.now();
|
|
7511
|
-
|
|
8430
|
+
// Manual work replaces the active poll that would otherwise be due. Move
|
|
8431
|
+
// that deadline before touching the governor so a data wake already in
|
|
8432
|
+
// this event-loop turn cannot start a second batch as soon as the forced
|
|
8433
|
+
// batch settles. The in-flight guard still coalesces work that overlaps.
|
|
8434
|
+
replaceActivePoll(kind, requestedAt);
|
|
7512
8435
|
const currentScope = ensureScope(requestedAt);
|
|
7513
8436
|
if (currentScope) {
|
|
7514
8437
|
heartbeatLease(currentScope, leaseId, tabRequestCost(key), requestedAt, key);
|
|
@@ -7538,7 +8461,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7538
8461
|
}
|
|
7539
8462
|
if (decision.value.status !== "scheduled" || decision.value.notBefore > nowMs) {
|
|
7540
8463
|
item.wasDeferred = true;
|
|
7541
|
-
setTabGovernorDecision(key, visibleGovernorDecision(
|
|
8464
|
+
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
8465
|
+
...decision.value,
|
|
8466
|
+
...sharedLaneProvenance(decision.value),
|
|
8467
|
+
}));
|
|
7542
8468
|
setTabWaiting(key, true);
|
|
7543
8469
|
armWake("data", decision.value.retryAt ?? decision.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7544
8470
|
continue;
|
|
@@ -7556,20 +8482,29 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7556
8482
|
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
7557
8483
|
...decision.value,
|
|
7558
8484
|
...started.value,
|
|
8485
|
+
...sharedLaneProvenance(decision.value),
|
|
7559
8486
|
}));
|
|
7560
8487
|
armWake("data", started.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7561
8488
|
continue;
|
|
7562
8489
|
}
|
|
7563
8490
|
setTabWaiting(key, false);
|
|
8491
|
+
replaceActivePoll(item.kind, nowMs);
|
|
7564
8492
|
if (item.force) coordinator.invalidate();
|
|
7565
8493
|
const descriptor = tabForKey(key);
|
|
7566
8494
|
void commit(
|
|
7567
8495
|
key,
|
|
7568
|
-
() => descriptor.fetch({
|
|
8496
|
+
() => descriptor.fetch({
|
|
8497
|
+
signal: controller.signal,
|
|
8498
|
+
runLimit: runLimitRef.current,
|
|
8499
|
+
entities: entityRef.current,
|
|
8500
|
+
force: item.force,
|
|
8501
|
+
previousRaw: rawRef.current[key] ?? null,
|
|
8502
|
+
}),
|
|
7569
8503
|
{
|
|
7570
8504
|
force: item.force,
|
|
7571
8505
|
scope: currentScope,
|
|
7572
8506
|
reservationId: decision.value.reservationId,
|
|
8507
|
+
admittedAt: nowMs,
|
|
7573
8508
|
automaticStatusVisible: item.wasDeferred && !item.force,
|
|
7574
8509
|
onSettled: () => finishPending(key, item.intentId),
|
|
7575
8510
|
},
|
|
@@ -7588,7 +8523,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7588
8523
|
const wakes = governorWakeTimes(snapshot.value, nowMs, runtime.refreshMs, leaseId);
|
|
7589
8524
|
const heldResources = Object.fromEntries(RATE_RESOURCES.map((resource) => {
|
|
7590
8525
|
const budget = snapshot.value.budgets[resource];
|
|
7591
|
-
const decision = budget
|
|
8526
|
+
const decision = budget
|
|
8527
|
+
? availableForGrant({ budget, resource, nowMs })
|
|
8528
|
+
: { mode: "paused" };
|
|
7592
8529
|
const held = !budget || budget.blockUntil > nowMs || decision.mode !== "open";
|
|
7593
8530
|
const retryAt = budget?.blockUntil > nowMs
|
|
7594
8531
|
? budget.blockUntil
|
|
@@ -7668,16 +8605,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7668
8605
|
attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
|
|
7669
8606
|
}
|
|
7670
8607
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7671
|
-
|
|
8608
|
+
const controlReady = pendingBlockPublications.size === 0 && governorControlReady(
|
|
7672
8609
|
refreshed,
|
|
7673
8610
|
snapshot,
|
|
7674
|
-
activeKey,
|
|
7675
8611
|
checkedAt,
|
|
7676
|
-
)
|
|
8612
|
+
);
|
|
8613
|
+
const activeEpochChanged = controlReady && controlEpochs !== null &&
|
|
8614
|
+
tabEpochChanged(controlEpochs, snapshot.value.epochs, activeKey);
|
|
8615
|
+
if (controlReady) controlEpochs = { ...snapshot.value.epochs };
|
|
8616
|
+
if (controlReady && (!liveScheduling || activeEpochChanged)) {
|
|
8617
|
+
const starting = !liveScheduling;
|
|
7677
8618
|
liveScheduling = true;
|
|
7678
8619
|
activePollAt = checkedAt + 1;
|
|
7679
|
-
|
|
7680
|
-
|
|
8620
|
+
if (starting) {
|
|
8621
|
+
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
8622
|
+
armWake("heartbeat", checkedAt + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
8623
|
+
}
|
|
7681
8624
|
}
|
|
7682
8625
|
if (!refreshed.ok) {
|
|
7683
8626
|
armWake("control", governorControlRetryAt(checkedAt, runtime.refreshMs), controlWake);
|
|
@@ -7714,12 +8657,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7714
8657
|
const snapshot = inspectGovernor(currentScope, checkedAt);
|
|
7715
8658
|
const activeKey = TABS[activeIndexRef.current].key;
|
|
7716
8659
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7717
|
-
|
|
8660
|
+
const controlReady = governorControlReady(
|
|
7718
8661
|
refreshed,
|
|
7719
8662
|
snapshot,
|
|
7720
|
-
activeKey,
|
|
7721
8663
|
checkedAt,
|
|
7722
|
-
)
|
|
8664
|
+
);
|
|
8665
|
+
if (controlReady) {
|
|
8666
|
+
controlEpochs = { ...snapshot.value.epochs };
|
|
7723
8667
|
liveScheduling = true;
|
|
7724
8668
|
activePollAt = checkedAt + 1;
|
|
7725
8669
|
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
@@ -7734,6 +8678,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7734
8678
|
void bootstrap();
|
|
7735
8679
|
return () => {
|
|
7736
8680
|
cancelled = true;
|
|
8681
|
+
queuedManual.clear();
|
|
8682
|
+
manualInFlight.clear();
|
|
7737
8683
|
wakeScheduler.clearAll();
|
|
7738
8684
|
for (const item of pending.values()) cancelIntent(cleanupScope, item.intentId, Date.now());
|
|
7739
8685
|
if (cleanupScope && registeredScopeHash) releaseLease(cleanupScope, leaseId);
|
|
@@ -7813,6 +8759,24 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7813
8759
|
|
|
7814
8760
|
const items = data[tab.key];
|
|
7815
8761
|
const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
|
|
8762
|
+
const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
|
|
8763
|
+
const coordinationReason = activeGovernorDecision?.reason ?? "unavailable";
|
|
8764
|
+
const coordinationCondition = coordinationError ? `${tab.key}\0${coordinationReason}` : null;
|
|
8765
|
+
useEffect(() => {
|
|
8766
|
+
setVisibleCoordinationCondition(null);
|
|
8767
|
+
if (coordinationCondition === null) return;
|
|
8768
|
+
const timer = setTimeout(
|
|
8769
|
+
() => setVisibleCoordinationCondition(coordinationCondition),
|
|
8770
|
+
COORDINATION_NOTICE_AFTER_MS,
|
|
8771
|
+
);
|
|
8772
|
+
return () => clearTimeout(timer);
|
|
8773
|
+
}, [coordinationCondition]);
|
|
8774
|
+
const showCoordinationNotice = coordinationCondition !== null &&
|
|
8775
|
+
visibleCoordinationCondition === coordinationCondition;
|
|
8776
|
+
const noticeLine = showCoordinationNotice
|
|
8777
|
+
? coordinationNotice(coordinationReason)
|
|
8778
|
+
: !remoteSetup && displayError ? displayError : "";
|
|
8779
|
+
const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
|
|
7816
8780
|
const spin = SPINNER[frame % SPINNER.length];
|
|
7817
8781
|
|
|
7818
8782
|
const counts = Object.fromEntries(
|
|
@@ -7867,6 +8831,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7867
8831
|
const staleAt = freshnessDeadline({
|
|
7868
8832
|
lastOk,
|
|
7869
8833
|
refreshMs: runtime.refreshMs,
|
|
8834
|
+
grantedMs: admittedCadenceRef.current[tab.key]?.grantedMs,
|
|
7870
8835
|
governorDecision: activeGovernorDecision,
|
|
7871
8836
|
currentEpochs: governorEpochs,
|
|
7872
8837
|
});
|
|
@@ -8075,20 +9040,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8075
9040
|
!showHelp &&
|
|
8076
9041
|
tooNarrow &&
|
|
8077
9042
|
e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
|
|
8078
|
-
|
|
8079
|
-
!tooNarrow &&
|
|
8080
|
-
!remoteSetup &&
|
|
8081
|
-
activeGovernorDecision?.coordinationError &&
|
|
8082
|
-
e(
|
|
8083
|
-
Text,
|
|
8084
|
-
{ color: ATTENTION, wrap: "truncate-end" },
|
|
8085
|
-
`API coordination unavailable (${activeGovernorDecision.reason}); retrying safely`,
|
|
8086
|
-
),
|
|
8087
|
-
!showHelp &&
|
|
8088
|
-
!tooNarrow &&
|
|
8089
|
-
!remoteSetup &&
|
|
8090
|
-
displayError &&
|
|
8091
|
-
e(Text, { color: ERROR_TEXT, wrap: "truncate-end" }, displayError),
|
|
9043
|
+
e(Text, { color: noticeTone, wrap: "truncate-end" }, noticeLine),
|
|
8092
9044
|
!showHelp &&
|
|
8093
9045
|
!tooNarrow &&
|
|
8094
9046
|
!remoteSetup &&
|
|
@@ -8167,7 +9119,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8167
9119
|
// Suppressed once icons are already ASCII, since then there is
|
|
8168
9120
|
// nothing to fix.
|
|
8169
9121
|
(USING_NERD_ICONS && iconHintDue ? " (icons blank? GH_GLANCE_ICONS=unicode)" : "")
|
|
8170
|
-
:
|
|
9122
|
+
: semanticStatus.kind === "paused" || waiting[tab.key]
|
|
8171
9123
|
? "waiting for API budget…"
|
|
8172
9124
|
: tab.key === "security" && securityNotes.length === ALERT_SOURCES.length
|
|
8173
9125
|
? "no alert sources available"
|
|
@@ -8184,6 +9136,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8184
9136
|
detail: activeGovernorDecision,
|
|
8185
9137
|
spin: showSpinner ? spin : null,
|
|
8186
9138
|
stale: staleLabel,
|
|
9139
|
+
nowMs: now.getTime(),
|
|
8187
9140
|
interactive,
|
|
8188
9141
|
cols: frameCols,
|
|
8189
9142
|
remoteSetup,
|
|
@@ -8461,6 +9414,9 @@ export {
|
|
|
8461
9414
|
BUDGET_SAFETY,
|
|
8462
9415
|
BUDGET_RESERVE_FRACTION,
|
|
8463
9416
|
BUDGET_SNAPSHOT_TTL_MS,
|
|
9417
|
+
GRAPHQL_BUDGET_SNAPSHOT_TTL_MS,
|
|
9418
|
+
budgetSnapshotTtl,
|
|
9419
|
+
currentSharedLaneProvenance,
|
|
8464
9420
|
GOVERNOR_HEARTBEAT_MS,
|
|
8465
9421
|
GOVERNOR_LEASE_TTL_MS,
|
|
8466
9422
|
GOVERNOR_PROBE_LEASE_MS,
|
|
@@ -8513,17 +9469,21 @@ export {
|
|
|
8513
9469
|
cancelIntent,
|
|
8514
9470
|
startReservation,
|
|
8515
9471
|
completeReservation,
|
|
9472
|
+
settleReservationWithBudgetObservations,
|
|
8516
9473
|
recordResourceBlock,
|
|
8517
9474
|
releaseLease,
|
|
8518
9475
|
inspectGovernor,
|
|
8519
9476
|
governorHealth,
|
|
8520
9477
|
refreshSharedBudget,
|
|
9478
|
+
readCoreBudget,
|
|
9479
|
+
readSharedBudgetSources,
|
|
8521
9480
|
safe,
|
|
8522
9481
|
shortErr,
|
|
8523
9482
|
isUnavailable,
|
|
8524
9483
|
isRateLimited,
|
|
8525
9484
|
isAuthProblem,
|
|
8526
9485
|
isMissingRemote,
|
|
9486
|
+
isUnusableOutput,
|
|
8527
9487
|
forwardSignalToChild,
|
|
8528
9488
|
toTabError,
|
|
8529
9489
|
formatTabError,
|
|
@@ -8540,6 +9500,19 @@ export {
|
|
|
8540
9500
|
BACKOFF_STEPS_MS,
|
|
8541
9501
|
redact,
|
|
8542
9502
|
classify,
|
|
9503
|
+
parseJsonOutput,
|
|
9504
|
+
parseGhApiResponse,
|
|
9505
|
+
pickRateLimit,
|
|
9506
|
+
ghApiArgs,
|
|
9507
|
+
ghApi,
|
|
9508
|
+
actionsRunsArgs,
|
|
9509
|
+
actionsWorkflowsArgs,
|
|
9510
|
+
parseActionsBodies,
|
|
9511
|
+
entityKey,
|
|
9512
|
+
fetchConditionalEntity,
|
|
9513
|
+
conditionalBatchResult,
|
|
9514
|
+
publishStagedEntities,
|
|
9515
|
+
fetchAlertSource,
|
|
8543
9516
|
formatAge,
|
|
8544
9517
|
formatDuration,
|
|
8545
9518
|
usableSize,
|
|
@@ -8580,6 +9553,8 @@ export {
|
|
|
8580
9553
|
pollSchedule,
|
|
8581
9554
|
retryPollAfterAdmissionFailure,
|
|
8582
9555
|
governorWakeTimes,
|
|
9556
|
+
governorControlReady,
|
|
9557
|
+
tabEpochChanged,
|
|
8583
9558
|
governorDataReady,
|
|
8584
9559
|
governorControlRetryAt,
|
|
8585
9560
|
createWakeScheduler,
|
|
@@ -8587,6 +9562,7 @@ export {
|
|
|
8587
9562
|
pendingFailureIsTerminal,
|
|
8588
9563
|
runtimeIntentGate,
|
|
8589
9564
|
rateLimitBlockDecision,
|
|
9565
|
+
coordinationNotice,
|
|
8590
9566
|
mergeRateLimitBlockPublications,
|
|
8591
9567
|
hydrateRateLimitBlockPublication,
|
|
8592
9568
|
retryRateLimitBlockPublication,
|
|
@@ -8622,8 +9598,10 @@ export {
|
|
|
8622
9598
|
summarizeDoctorEnv,
|
|
8623
9599
|
widthStatusText,
|
|
8624
9600
|
refreshStatus,
|
|
9601
|
+
statusInterval,
|
|
8625
9602
|
statusBarLayout,
|
|
8626
9603
|
freshnessDeadline,
|
|
9604
|
+
nextAdmittedCadence,
|
|
8627
9605
|
probingGovernorDecisions,
|
|
8628
9606
|
retainDeferredGovernorHold,
|
|
8629
9607
|
REFRESH_STATUS_GLYPHS,
|
|
@@ -8642,4 +9620,5 @@ export {
|
|
|
8642
9620
|
REMOTE_SETUP_NONINTERACTIVE_LINES,
|
|
8643
9621
|
VERDICT_REMEDY,
|
|
8644
9622
|
FAILURE_LADDER,
|
|
9623
|
+
COORDINATION_NOTICE_AFTER_MS,
|
|
8645
9624
|
};
|