gh-glance 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -15
- package/README.md +31 -20
- package/index.mjs +1207 -240
- package/package.json +1 -1
package/index.mjs
CHANGED
|
@@ -162,6 +162,11 @@ const AUTH_RETRY_MS = [30_000];
|
|
|
162
162
|
// byte-identical-idle-frame property the rest of this file works to keep.
|
|
163
163
|
const STALE_AFTER_MS = 30_000;
|
|
164
164
|
|
|
165
|
+
// A coordination hold must persist long enough to be meaningful before it
|
|
166
|
+
// takes the notice row. Short startup and lock-contention states recover on
|
|
167
|
+
// their own and should not present themselves as user-visible failures.
|
|
168
|
+
const COORDINATION_NOTICE_AFTER_MS = 2_000;
|
|
169
|
+
|
|
165
170
|
// Mutable because the fetchers below are defined before argv is parsed, and the
|
|
166
171
|
// argv block is what fills this in. Everything here has a working default, so
|
|
167
172
|
// the zero-argument invocation the README documents behaves exactly as before.
|
|
@@ -390,6 +395,24 @@ function isMissingRemote(err) {
|
|
|
390
395
|
return /(?:failed to determine base repo:\s*)?no git remotes found/i.test(errText(err));
|
|
391
396
|
}
|
|
392
397
|
|
|
398
|
+
const UNUSABLE_OUTPUT_CODE = "GH_GLANCE_UNUSABLE_OUTPUT";
|
|
399
|
+
|
|
400
|
+
// Only JSON parsed from a successful gh stdout stream goes through this seam.
|
|
401
|
+
// Marking the SyntaxError here distinguishes our own failed parse from a gh
|
|
402
|
+
// stderr message that happens to use the same words.
|
|
403
|
+
function parseJsonOutput(raw) {
|
|
404
|
+
try {
|
|
405
|
+
return JSON.parse(raw);
|
|
406
|
+
} catch (error) {
|
|
407
|
+
if (error instanceof SyntaxError) error.code = UNUSABLE_OUTPUT_CODE;
|
|
408
|
+
throw error;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function isUnusableOutput(err) {
|
|
413
|
+
return err?.code === UNUSABLE_OUTPUT_CODE;
|
|
414
|
+
}
|
|
415
|
+
|
|
393
416
|
// The one place the three predicates are turned into a verdict. Both consumers
|
|
394
417
|
// go through it -- the fetcher, to choose a note and a backoff ladder, and
|
|
395
418
|
// `--doctor`, to report what gh-glance concluded -- so the report cannot claim
|
|
@@ -409,6 +432,7 @@ function classify(err) {
|
|
|
409
432
|
if (isRateLimited(err)) return "rate-limited";
|
|
410
433
|
if (isAuthProblem(err)) return "auth-problem";
|
|
411
434
|
if (isUnavailable(err)) return "unavailable";
|
|
435
|
+
if (isUnusableOutput(err)) return "unusable-output";
|
|
412
436
|
return "other";
|
|
413
437
|
}
|
|
414
438
|
|
|
@@ -624,6 +648,92 @@ async function runGh(args, { signal, operation } = {}) {
|
|
|
624
648
|
}
|
|
625
649
|
}
|
|
626
650
|
|
|
651
|
+
function parseGhApiResponse(stdout) {
|
|
652
|
+
const text = typeof stdout === "string" ? stdout : String(stdout ?? "");
|
|
653
|
+
const separator = /\r?\n\r?\n/.exec(text);
|
|
654
|
+
const prefix = separator ? text.slice(0, separator.index) : text;
|
|
655
|
+
const body = separator ? text.slice(separator.index + separator[0].length) : "";
|
|
656
|
+
const lines = prefix.split(/\r?\n/);
|
|
657
|
+
const statusMatch = /^HTTP\/\d(?:\.\d+)?\s+(\d{3})(?:\s|$)/i.exec(lines[0] ?? "");
|
|
658
|
+
const headers = {};
|
|
659
|
+
for (const line of lines.slice(1)) {
|
|
660
|
+
const colon = line.indexOf(":");
|
|
661
|
+
if (colon <= 0) continue;
|
|
662
|
+
const name = line.slice(0, colon).trim().toLowerCase();
|
|
663
|
+
if (name) headers[name] = line.slice(colon + 1).trim();
|
|
664
|
+
}
|
|
665
|
+
return {
|
|
666
|
+
status: statusMatch ? Number(statusMatch[1]) : null,
|
|
667
|
+
headers,
|
|
668
|
+
body,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function pickRateLimit(headers) {
|
|
673
|
+
if (!headers || typeof headers !== "object" || Array.isArray(headers)) return null;
|
|
674
|
+
const integer = (name) => {
|
|
675
|
+
const value = Number(headers[name]);
|
|
676
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
677
|
+
};
|
|
678
|
+
const resource = typeof headers["x-ratelimit-resource"] === "string"
|
|
679
|
+
? headers["x-ratelimit-resource"].trim().toLowerCase()
|
|
680
|
+
: "";
|
|
681
|
+
const limit = integer("x-ratelimit-limit");
|
|
682
|
+
const used = integer("x-ratelimit-used");
|
|
683
|
+
const remaining = integer("x-ratelimit-remaining");
|
|
684
|
+
const reset = integer("x-ratelimit-reset");
|
|
685
|
+
if (!RATE_RESOURCES.includes(resource) || [limit, used, remaining, reset].includes(null)) return null;
|
|
686
|
+
return { resource, limit, used, remaining, resetMs: reset * 1000 };
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
function responseHeaderObservation(rateLimit, cost, receivedAt = Date.now()) {
|
|
690
|
+
return rateLimit ? {
|
|
691
|
+
...rateLimit,
|
|
692
|
+
source: "response-header",
|
|
693
|
+
receivedAt,
|
|
694
|
+
cost,
|
|
695
|
+
} : null;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function ghApiArgs(args, etag = null) {
|
|
699
|
+
return [
|
|
700
|
+
"api",
|
|
701
|
+
"-i",
|
|
702
|
+
...args,
|
|
703
|
+
...(typeof etag === "string" && etag.length > 0 ? ["-H", `If-None-Match: ${etag}`] : []),
|
|
704
|
+
];
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
async function ghApi(args, { operation, signal, etag = null, run = runGh } = {}) {
|
|
708
|
+
const argv = ghApiArgs(args, etag);
|
|
709
|
+
try {
|
|
710
|
+
const parsed = parseGhApiResponse(await run(argv, { operation, signal }));
|
|
711
|
+
return {
|
|
712
|
+
status: parsed.status,
|
|
713
|
+
etag: parsed.headers.etag ?? null,
|
|
714
|
+
rateLimit: pickRateLimit(parsed.headers),
|
|
715
|
+
body: parsed.body,
|
|
716
|
+
};
|
|
717
|
+
} catch (error) {
|
|
718
|
+
const parsed = parseGhApiResponse(error?.stdout);
|
|
719
|
+
if (parsed.status === 304) {
|
|
720
|
+
return {
|
|
721
|
+
status: 304,
|
|
722
|
+
etag: parsed.headers.etag ?? null,
|
|
723
|
+
rateLimit: pickRateLimit(parsed.headers),
|
|
724
|
+
body: "",
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
if (parsed.status !== null) {
|
|
728
|
+
error.apiResponse = parsed;
|
|
729
|
+
const rateLimit = pickRateLimit(parsed.headers);
|
|
730
|
+
const observation = responseHeaderObservation(rateLimit, 0);
|
|
731
|
+
if (observation) error.budgetObservations = [observation];
|
|
732
|
+
}
|
|
733
|
+
throw error;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
627
737
|
// The target as `gh` spells it: host-qualified when a host was given, the bare
|
|
628
738
|
// slug otherwise -- which is exactly the [HOST/]OWNER/REPO form `gh --repo`
|
|
629
739
|
// documents.
|
|
@@ -872,40 +982,161 @@ function createFailureContextCoordinator({ resolve, commit, fallback }) {
|
|
|
872
982
|
return { ensure, invalidate };
|
|
873
983
|
}
|
|
874
984
|
|
|
875
|
-
function
|
|
985
|
+
function actionsRunsArgs(limit) {
|
|
876
986
|
return [
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
"
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
987
|
+
apiPath(`repos/{owner}/{repo}/actions/runs?exclude_pull_requests=true&per_page=${limit}`),
|
|
988
|
+
...apiHostArgs(),
|
|
989
|
+
"--jq",
|
|
990
|
+
"[.workflow_runs[] | {databaseId: .id, displayTitle: .display_title, number: .run_number, headBranch: .head_branch, status, conclusion, startedAt: .run_started_at, updatedAt: .updated_at, workflowId: .workflow_id}]",
|
|
991
|
+
];
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
function actionsWorkflowsArgs() {
|
|
995
|
+
return [
|
|
996
|
+
apiPath("repos/{owner}/{repo}/actions/workflows?page=1&per_page=100"),
|
|
997
|
+
...apiHostArgs(),
|
|
998
|
+
"--jq",
|
|
999
|
+
"[.workflows[] | {id, name}]",
|
|
884
1000
|
];
|
|
885
1001
|
}
|
|
886
1002
|
|
|
887
|
-
|
|
888
|
-
const
|
|
1003
|
+
function parseActionsBodies(runsBody, workflowsBody) {
|
|
1004
|
+
const workflows = new Map(parseJsonOutput(workflowsBody).map((workflow) => [
|
|
1005
|
+
workflow.id,
|
|
1006
|
+
safe(workflow.name),
|
|
1007
|
+
]));
|
|
1008
|
+
return parseJsonOutput(runsBody).map((run) => ({
|
|
1009
|
+
databaseId: run.databaseId,
|
|
1010
|
+
displayTitle: safe(run.displayTitle),
|
|
1011
|
+
workflowName: workflows.get(run.workflowId) ?? "",
|
|
1012
|
+
number: run.number,
|
|
1013
|
+
headBranch: safe(run.headBranch),
|
|
1014
|
+
status: run.status,
|
|
1015
|
+
conclusion: run.conclusion,
|
|
1016
|
+
startedAt: run.startedAt,
|
|
1017
|
+
updatedAt: run.updatedAt,
|
|
1018
|
+
}));
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
function entityKey(tab, path) {
|
|
1022
|
+
return `${tab}\0${path}`;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
function cachedEntity(entities, tab, path) {
|
|
1026
|
+
const value = entities instanceof Map ? entities.get(entityKey(tab, path)) : null;
|
|
1027
|
+
return typeof value?.etag === "string" && typeof value?.body === "string" ? value : null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
async function fetchConditionalEntity({
|
|
1031
|
+
tab,
|
|
1032
|
+
args,
|
|
1033
|
+
operation,
|
|
1034
|
+
signal,
|
|
1035
|
+
force,
|
|
1036
|
+
entities,
|
|
1037
|
+
request = ghApi,
|
|
1038
|
+
}) {
|
|
1039
|
+
const path = args[0];
|
|
1040
|
+
const cached = cachedEntity(entities, tab, path);
|
|
1041
|
+
const response = await request(args, {
|
|
1042
|
+
operation,
|
|
1043
|
+
signal,
|
|
1044
|
+
etag: force ? null : cached?.etag ?? null,
|
|
1045
|
+
});
|
|
1046
|
+
const observation = responseHeaderObservation(
|
|
1047
|
+
response.rateLimit,
|
|
1048
|
+
response.status === 200 ? 1 : 0,
|
|
1049
|
+
);
|
|
1050
|
+
if (response.status === 304) {
|
|
1051
|
+
return {
|
|
1052
|
+
path,
|
|
1053
|
+
status: 304,
|
|
1054
|
+
body: cached?.body ?? null,
|
|
1055
|
+
staged: null,
|
|
1056
|
+
observations: observation ? [observation] : [],
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
889
1059
|
return {
|
|
890
|
-
|
|
1060
|
+
path,
|
|
1061
|
+
status: response.status,
|
|
1062
|
+
body: response.body,
|
|
1063
|
+
staged: response.status === 200
|
|
1064
|
+
? [entityKey(tab, path), typeof response.etag === "string"
|
|
1065
|
+
? { etag: response.etag, body: response.body }
|
|
1066
|
+
: null]
|
|
1067
|
+
: null,
|
|
1068
|
+
observations: observation ? [observation] : [],
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function conditionalBatchResult(responses, previousRaw, joinedRaw = null) {
|
|
1073
|
+
const list = Array.isArray(responses) ? responses : [];
|
|
1074
|
+
let allNotModified = list.length > 0;
|
|
1075
|
+
let restSpent = 0;
|
|
1076
|
+
let stagedEntities = null;
|
|
1077
|
+
const observations = [];
|
|
1078
|
+
for (const response of list) {
|
|
1079
|
+
if (response?.status !== 304 || typeof response.body !== "string") allNotModified = false;
|
|
1080
|
+
if (response?.status === 200) restSpent += 1;
|
|
1081
|
+
if (Array.isArray(response?.observations)) observations.push(...response.observations);
|
|
1082
|
+
if (!response?.staged) continue;
|
|
1083
|
+
stagedEntities ??= new Map();
|
|
1084
|
+
stagedEntities.set(...response.staged);
|
|
1085
|
+
}
|
|
1086
|
+
return {
|
|
1087
|
+
raw: allNotModified
|
|
1088
|
+
? previousRaw
|
|
1089
|
+
: joinedRaw ?? list.map((response) => response?.body ?? "").join("\0"),
|
|
1090
|
+
allNotModified,
|
|
1091
|
+
restSpent,
|
|
1092
|
+
stagedEntities,
|
|
1093
|
+
observations,
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function publishStagedEntities(entities, staged, transitionKind) {
|
|
1098
|
+
if (!(entities instanceof Map) || !(staged instanceof Map) ||
|
|
1099
|
+
!["changed", "unchanged"].includes(transitionKind)) return false;
|
|
1100
|
+
for (const [key, value] of staged) {
|
|
1101
|
+
if (value === null) entities.delete(key);
|
|
1102
|
+
else entities.set(key, value);
|
|
1103
|
+
}
|
|
1104
|
+
return true;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
async function fetchActions(limit, signal, {
|
|
1108
|
+
entities = new Map(),
|
|
1109
|
+
force = false,
|
|
1110
|
+
previousRaw = null,
|
|
1111
|
+
} = {}) {
|
|
1112
|
+
const responses = await Promise.all([
|
|
1113
|
+
fetchConditionalEntity({
|
|
1114
|
+
tab: "actions",
|
|
1115
|
+
args: actionsRunsArgs(limit),
|
|
1116
|
+
operation: "tab:actions-runs",
|
|
1117
|
+
signal,
|
|
1118
|
+
force,
|
|
1119
|
+
entities,
|
|
1120
|
+
}),
|
|
1121
|
+
fetchConditionalEntity({
|
|
1122
|
+
tab: "actions",
|
|
1123
|
+
args: actionsWorkflowsArgs(),
|
|
1124
|
+
operation: "tab:actions-workflows",
|
|
1125
|
+
signal,
|
|
1126
|
+
force,
|
|
1127
|
+
entities,
|
|
1128
|
+
}),
|
|
1129
|
+
]);
|
|
1130
|
+
const [runs, workflows] = responses;
|
|
1131
|
+
const batch = conditionalBatchResult(responses, previousRaw);
|
|
1132
|
+
return {
|
|
1133
|
+
raw: batch.raw,
|
|
891
1134
|
limit,
|
|
892
|
-
|
|
893
|
-
// re-derived from the argv shape -- a second copy of the cost model is the
|
|
894
|
-
// drift this file warns about repeatedly.
|
|
895
|
-
restSpent: REST_PER_FETCH.actions,
|
|
1135
|
+
restSpent: batch.restSpent,
|
|
896
1136
|
graphqlSpent: GRAPHQL_PER_FETCH.actions,
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
displayTitle: safe(r.displayTitle),
|
|
901
|
-
workflowName: safe(r.workflowName),
|
|
902
|
-
number: r.number,
|
|
903
|
-
headBranch: safe(r.headBranch),
|
|
904
|
-
status: r.status,
|
|
905
|
-
conclusion: r.conclusion,
|
|
906
|
-
startedAt: r.startedAt,
|
|
907
|
-
updatedAt: r.updatedAt,
|
|
908
|
-
})),
|
|
1137
|
+
stagedEntities: batch.stagedEntities,
|
|
1138
|
+
observations: batch.observations,
|
|
1139
|
+
parse: () => parseActionsBodies(runs.body ?? "", workflows.body ?? ""),
|
|
909
1140
|
};
|
|
910
1141
|
}
|
|
911
1142
|
|
|
@@ -944,7 +1175,7 @@ async function fetchIssues(signal) {
|
|
|
944
1175
|
restSpent: REST_PER_FETCH.issues,
|
|
945
1176
|
graphqlSpent: GRAPHQL_PER_FETCH.issues,
|
|
946
1177
|
parse: () =>
|
|
947
|
-
|
|
1178
|
+
parseJsonOutput(raw).map((i) => ({
|
|
948
1179
|
number: i.number,
|
|
949
1180
|
title: safe(i.title),
|
|
950
1181
|
author: safe(i.author?.login ?? ""),
|
|
@@ -978,7 +1209,7 @@ async function fetchPRs(signal) {
|
|
|
978
1209
|
restSpent: REST_PER_FETCH.prs,
|
|
979
1210
|
graphqlSpent: GRAPHQL_PER_FETCH.prs,
|
|
980
1211
|
parse: () =>
|
|
981
|
-
|
|
1212
|
+
parseJsonOutput(raw).map((p) => ({
|
|
982
1213
|
number: p.number,
|
|
983
1214
|
title: safe(p.title),
|
|
984
1215
|
author: safe(p.author?.login ?? ""),
|
|
@@ -1086,6 +1317,8 @@ function tabRequestCost(tab) {
|
|
|
1086
1317
|
// individual calls covered by the tab's six-call upfront reservation.
|
|
1087
1318
|
const OPERATION_COSTS = Object.freeze({
|
|
1088
1319
|
...Object.fromEntries(TAB_KEYS.map((tab) => [`tab:${tab}`, tabRequestCost(tab)])),
|
|
1320
|
+
"tab:actions-runs": { core: 1, graphql: 0 },
|
|
1321
|
+
"tab:actions-workflows": { core: 1, graphql: 0 },
|
|
1089
1322
|
"tab:security-endpoint": { core: 1, graphql: 0 },
|
|
1090
1323
|
"failure-context:repository": { core: 0, graphql: 1 },
|
|
1091
1324
|
"failure-context:auth": { core: 0, graphql: 0 },
|
|
@@ -1093,10 +1326,15 @@ const OPERATION_COSTS = Object.freeze({
|
|
|
1093
1326
|
"open:issues": { core: 0, graphql: GRAPHQL_PER_FETCH.issues },
|
|
1094
1327
|
"open:prs": { core: 0, graphql: GRAPHQL_PER_FETCH.prs },
|
|
1095
1328
|
"doctor:repository": { core: 0, graphql: 1 },
|
|
1096
|
-
"doctor:actions":
|
|
1329
|
+
"doctor:actions-runs": { core: 1, graphql: 0 },
|
|
1330
|
+
"doctor:actions-workflows": { core: 1, graphql: 0 },
|
|
1097
1331
|
"doctor:issues": tabRequestCost("issues"),
|
|
1098
1332
|
"doctor:prs": tabRequestCost("prs"),
|
|
1099
1333
|
"doctor:security-endpoint": { core: 1, graphql: 0 },
|
|
1334
|
+
// The claimed observer is the one control-plane exception to normal data
|
|
1335
|
+
// admission, but its worst-case vector is still one core unit: the first 200
|
|
1336
|
+
// can spend it before an ETag makes later observations free.
|
|
1337
|
+
"budget-core-observer": { core: 1, graphql: 0 },
|
|
1100
1338
|
"rate-limit": { core: 0, graphql: 0 },
|
|
1101
1339
|
"version": { core: 0, graphql: 0 },
|
|
1102
1340
|
"auth-status": { core: 0, graphql: 0 },
|
|
@@ -1111,7 +1349,15 @@ function operationCost(operation) {
|
|
|
1111
1349
|
// is for the user's own `gh` and `git` commands.
|
|
1112
1350
|
const BUDGET_SAFETY = 0.8;
|
|
1113
1351
|
const BUDGET_RESERVE_FRACTION = 1 - BUDGET_SAFETY;
|
|
1352
|
+
// How often to re-read the budget. `gh api rate_limit` does not count against
|
|
1353
|
+
// the limit (verified, delta 0 -- see rateBudget) but it is still a subprocess,
|
|
1354
|
+
// so once a minute rather than once a tick: the quantity it measures moves on
|
|
1355
|
+
// the scale of minutes.
|
|
1356
|
+
const BUDGET_PROBE_MS = 60_000;
|
|
1114
1357
|
const BUDGET_SNAPSHOT_TTL_MS = 65_000;
|
|
1358
|
+
// GraphQL has no response-header owner. One unusable minute sample must not
|
|
1359
|
+
// close admission, but two consecutive misses still do: 2 probes + 5s grace.
|
|
1360
|
+
const GRAPHQL_BUDGET_SNAPSHOT_TTL_MS = 2 * BUDGET_PROBE_MS + 5_000;
|
|
1115
1361
|
const GOVERNOR_HEARTBEAT_MS = 20_000;
|
|
1116
1362
|
const GOVERNOR_LEASE_TTL_MS = 90_000;
|
|
1117
1363
|
const GOVERNOR_PROBE_LEASE_MS = 70_000;
|
|
@@ -1120,11 +1366,27 @@ const BUDGET_RESET_GRACE_MS = 2_000;
|
|
|
1120
1366
|
const GOVERNOR_PHASE_WINDOW_MS = 5_000;
|
|
1121
1367
|
const BUDGET_WINDOW_MS = 3_600_000;
|
|
1122
1368
|
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1369
|
+
function budgetSnapshotTtl(resource) {
|
|
1370
|
+
return resource === "graphql" ? GRAPHQL_BUDGET_SNAPSHOT_TTL_MS : BUDGET_SNAPSHOT_TTL_MS;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
function sharedLaneProvenance(value) {
|
|
1374
|
+
return value?.waitCause === "shared-lane" &&
|
|
1375
|
+
Number.isSafeInteger(value?.sharingCount) && value.sharingCount > 1
|
|
1376
|
+
? { waitCause: "shared-lane", sharingCount: value.sharingCount }
|
|
1377
|
+
: {};
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
function sharedLaneEvidence(value) {
|
|
1381
|
+
const provenance = sharedLaneProvenance(value);
|
|
1382
|
+
const ownerLeaseIds = Array.isArray(value?.sharingOwnerLeaseIds)
|
|
1383
|
+
? [...new Set(value.sharingOwnerLeaseIds.filter((leaseId) =>
|
|
1384
|
+
typeof leaseId === "string" && leaseId.length > 0))]
|
|
1385
|
+
: [];
|
|
1386
|
+
return provenance.waitCause && ownerLeaseIds.length > 0
|
|
1387
|
+
? { ...provenance, sharingOwnerLeaseIds: ownerLeaseIds }
|
|
1388
|
+
: {};
|
|
1389
|
+
}
|
|
1128
1390
|
|
|
1129
1391
|
// Below this many completed shared calls in a probe window, external-spend
|
|
1130
1392
|
// inference is noise. Under the threshold the loop retains the previous factor.
|
|
@@ -1175,6 +1437,23 @@ function leaseFor(leases, leaseId) {
|
|
|
1175
1437
|
return leases instanceof Map ? leases.get(leaseId) : leases?.[leaseId];
|
|
1176
1438
|
}
|
|
1177
1439
|
|
|
1440
|
+
function countLiveLeases(leases, nowMs) {
|
|
1441
|
+
let count = 0;
|
|
1442
|
+
const values = leases instanceof Map ? leases.values() : Object.values(leases);
|
|
1443
|
+
for (const lease of values) {
|
|
1444
|
+
if (Number.isFinite(lease?.expiresAt) && lease.expiresAt > nowMs) count += 1;
|
|
1445
|
+
}
|
|
1446
|
+
return count;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
function currentSharedLaneProvenance(value, leases, nowMs) {
|
|
1450
|
+
const evidence = sharedLaneEvidence(value);
|
|
1451
|
+
if (!evidence.waitCause || countLiveLeases(leases, nowMs) !== evidence.sharingCount) return {};
|
|
1452
|
+
return evidence.sharingOwnerLeaseIds.every((leaseId) => leaseFor(leases, leaseId)?.expiresAt > nowMs)
|
|
1453
|
+
? sharedLaneProvenance(evidence)
|
|
1454
|
+
: {};
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1178
1457
|
function reservationCost(reservation, resource, leases, nowMs) {
|
|
1179
1458
|
if (!reservation || ["cancelled", "reconciled"].includes(reservation.status)) return 0;
|
|
1180
1459
|
if (reservation.status === "scheduled") {
|
|
@@ -1187,11 +1466,13 @@ function reservationCost(reservation, resource, leases, nowMs) {
|
|
|
1187
1466
|
: reservation.costs;
|
|
1188
1467
|
const cost = chargedCosts?.[resource] ??
|
|
1189
1468
|
(reservation.resource === resource ? reservation.cost : 0);
|
|
1190
|
-
|
|
1469
|
+
const accounted = reservation.accountedCosts?.[resource] ?? 0;
|
|
1470
|
+
return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
|
|
1191
1471
|
}
|
|
1192
1472
|
|
|
1193
1473
|
function availableForGrant({
|
|
1194
1474
|
budget,
|
|
1475
|
+
resource = budget?.resource,
|
|
1195
1476
|
reservations = [],
|
|
1196
1477
|
leases = {},
|
|
1197
1478
|
nowMs,
|
|
@@ -1199,8 +1480,9 @@ function availableForGrant({
|
|
|
1199
1480
|
}) {
|
|
1200
1481
|
const normalized = normalizeBudgetResource(budget);
|
|
1201
1482
|
if (!normalized) return { mode: "probe", reason: "budget-unknown" };
|
|
1483
|
+
if (!RATE_RESOURCES.includes(resource)) return { mode: "probe", reason: "budget-resource" };
|
|
1202
1484
|
if (normalized.observedAt > nowMs) return { mode: "probe", reason: "budget-future" };
|
|
1203
|
-
if (nowMs - normalized.observedAt >
|
|
1485
|
+
if (nowMs - normalized.observedAt > budgetSnapshotTtl(resource)) {
|
|
1204
1486
|
return { mode: "probe", reason: "budget-stale" };
|
|
1205
1487
|
}
|
|
1206
1488
|
if (nowMs >= normalized.resetMs) return { mode: "probe", reason: "budget-reset" };
|
|
@@ -1226,7 +1508,7 @@ function availableForGrant({
|
|
|
1226
1508
|
|
|
1227
1509
|
const reserve = resourceReserve(normalized.limit);
|
|
1228
1510
|
const charged = chargedCost ?? reservations.reduce(
|
|
1229
|
-
(total, reservation) => total + reservationCost(reservation,
|
|
1511
|
+
(total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
|
|
1230
1512
|
0,
|
|
1231
1513
|
);
|
|
1232
1514
|
return {
|
|
@@ -1272,7 +1554,8 @@ function resourceDecision({
|
|
|
1272
1554
|
chargedCost = null,
|
|
1273
1555
|
}) {
|
|
1274
1556
|
const capacity = availableForGrant({
|
|
1275
|
-
budget
|
|
1557
|
+
budget,
|
|
1558
|
+
resource,
|
|
1276
1559
|
reservations,
|
|
1277
1560
|
leases,
|
|
1278
1561
|
nowMs,
|
|
@@ -1480,6 +1763,7 @@ function scheduleIntents({
|
|
|
1480
1763
|
const updatedCursors = { ...cursors };
|
|
1481
1764
|
const grants = [];
|
|
1482
1765
|
const denied = [];
|
|
1766
|
+
const liveLeaseCount = countLiveLeases(leases, nowMs);
|
|
1483
1767
|
const priorities = [...new Set(valid.map((intent) => intent.normalizedPriority))].sort((a, b) => a - b);
|
|
1484
1768
|
|
|
1485
1769
|
for (const priority of priorities) {
|
|
@@ -1510,9 +1794,9 @@ function scheduleIntents({
|
|
|
1510
1794
|
resource,
|
|
1511
1795
|
governorEpochPhaseAt(intent.phaseSeed, decisions[resource]),
|
|
1512
1796
|
]));
|
|
1797
|
+
const intrinsicAt = Math.max(nowMs, ...Object.values(phaseTimes));
|
|
1513
1798
|
const notBefore = Math.max(
|
|
1514
|
-
|
|
1515
|
-
...Object.values(phaseTimes),
|
|
1799
|
+
intrinsicAt,
|
|
1516
1800
|
...resources.map((resource) => updatedLanes[resource]?.nextAt ?? nowMs),
|
|
1517
1801
|
);
|
|
1518
1802
|
const expiring = resources.find((resource) => notBefore >= decisions[resource].resetMs);
|
|
@@ -1538,6 +1822,18 @@ function scheduleIntents({
|
|
|
1538
1822
|
status: "scheduled",
|
|
1539
1823
|
epochs: Object.fromEntries(resources.map((resource) => [resource, decisions[resource].epoch])),
|
|
1540
1824
|
};
|
|
1825
|
+
const limitingLanes = resources.filter((resource) =>
|
|
1826
|
+
updatedLanes[resource]?.nextAt === notBefore);
|
|
1827
|
+
const sharingOwnerLeaseIds = [...new Set(limitingLanes.map((resource) =>
|
|
1828
|
+
updatedCursors[resource]))];
|
|
1829
|
+
const sharedLane = liveLeaseCount > 1 && notBefore > intrinsicAt &&
|
|
1830
|
+
sharingOwnerLeaseIds.length > 0 && sharingOwnerLeaseIds.every((owner) =>
|
|
1831
|
+
owner !== intent.leaseId && leaseFor(leases, owner)?.expiresAt > nowMs);
|
|
1832
|
+
if (sharedLane) {
|
|
1833
|
+
reservation.waitCause = "shared-lane";
|
|
1834
|
+
reservation.sharingCount = liveLeaseCount;
|
|
1835
|
+
reservation.sharingOwnerLeaseIds = sharingOwnerLeaseIds;
|
|
1836
|
+
}
|
|
1541
1837
|
grants.push(reservation);
|
|
1542
1838
|
for (const resource of resources) {
|
|
1543
1839
|
chargedTotals[resource] += intent.costs[resource];
|
|
@@ -1562,7 +1858,11 @@ function scheduleIntents({
|
|
|
1562
1858
|
|
|
1563
1859
|
// ---------- Shared account governor ----------
|
|
1564
1860
|
|
|
1565
|
-
|
|
1861
|
+
// The scope version is deliberately stable. Stored protocol revisions must use
|
|
1862
|
+
// the same file so an older process sees the new exact shape and fails closed
|
|
1863
|
+
// instead of coordinating through a second file.
|
|
1864
|
+
const GOVERNOR_SCOPE_VERSION = 1;
|
|
1865
|
+
const GOVERNOR_STATE_VERSION = 2;
|
|
1566
1866
|
const GOVERNOR_MAX_LEASES = 128;
|
|
1567
1867
|
const GOVERNOR_MAX_INTENTS = 512;
|
|
1568
1868
|
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
@@ -1571,6 +1871,7 @@ const GOVERNOR_PROBE_DRAIN_MS = 30_000;
|
|
|
1571
1871
|
const GOVERNOR_PUBLICATION_REINSPECT_MS = 1_000;
|
|
1572
1872
|
const GOVERNOR_PROBE_TRANSITION_MS = 5_000;
|
|
1573
1873
|
const GOVERNOR_MAX_FUTURE_MS = 24 * 60 * 60 * 1000;
|
|
1874
|
+
const persistenceWaitCell = new Int32Array(new SharedArrayBuffer(4));
|
|
1574
1875
|
const GOVERNOR_OUTCOMES = new Set([
|
|
1575
1876
|
"measured-success",
|
|
1576
1877
|
"rejected",
|
|
@@ -1605,7 +1906,7 @@ function governorScopeHash(effectiveHost, authIdentity) {
|
|
|
1605
1906
|
const host = normalizeHost(effectiveHost);
|
|
1606
1907
|
if (!host || typeof authIdentity !== "string" || authIdentity.length === 0) return null;
|
|
1607
1908
|
return createHash("sha256")
|
|
1608
|
-
.update(JSON.stringify({ version:
|
|
1909
|
+
.update(JSON.stringify({ version: GOVERNOR_SCOPE_VERSION, effectiveHost: host, authCacheIdentity: authIdentity }))
|
|
1609
1910
|
.digest("hex");
|
|
1610
1911
|
}
|
|
1611
1912
|
|
|
@@ -1673,6 +1974,9 @@ function emptyGovernorState() {
|
|
|
1673
1974
|
version: GOVERNOR_STATE_VERSION,
|
|
1674
1975
|
epochs: { core: null, graphql: null },
|
|
1675
1976
|
budgets: {},
|
|
1977
|
+
observers: {
|
|
1978
|
+
core: { etag: null, outcome: "idle", at: 0, nextAt: 0 },
|
|
1979
|
+
},
|
|
1676
1980
|
probeClaim: null,
|
|
1677
1981
|
probeOutcome: { status: "idle", at: 0, nextAt: 0 },
|
|
1678
1982
|
leases: {},
|
|
@@ -1693,10 +1997,11 @@ function exactKeys(value, keys) {
|
|
|
1693
1997
|
keys.every((key) => Object.hasOwn(value, key));
|
|
1694
1998
|
}
|
|
1695
1999
|
|
|
1696
|
-
function normalizeGovernorBudget(raw, nowMs) {
|
|
2000
|
+
function normalizeGovernorBudget(raw, nowMs, resource = null) {
|
|
1697
2001
|
if (!exactKeys(raw, [
|
|
1698
2002
|
"limit", "remaining", "used", "resetMs", "observedAt", "blockUntil",
|
|
1699
2003
|
"blockReason", "laneNextAt", "roundRobinCursor", "lastExternalFactor", "epoch",
|
|
2004
|
+
"source", "factorBaseline", "knownLocalUsed",
|
|
1700
2005
|
])) return null;
|
|
1701
2006
|
const normalized = normalizeBudgetResource(raw);
|
|
1702
2007
|
if (
|
|
@@ -1708,6 +2013,14 @@ function normalizeGovernorBudget(raw, nowMs) {
|
|
|
1708
2013
|
(raw.blockReason !== null && !GOVERNOR_BLOCK_REASONS.has(raw.blockReason)) ||
|
|
1709
2014
|
(raw.roundRobinCursor !== null && !validGovernorId(raw.roundRobinCursor)) ||
|
|
1710
2015
|
!Number.isFinite(raw.lastExternalFactor) || raw.lastExternalFactor < 1 ||
|
|
2016
|
+
!["response-header", "core-observer", "rate-limit-probe"].includes(raw.source) ||
|
|
2017
|
+
raw.source === "rate-limit-probe" && resource === "core" ||
|
|
2018
|
+
!exactKeys(raw.factorBaseline, ["epoch", "used", "observedAt"]) ||
|
|
2019
|
+
!validGovernorEpoch(raw.factorBaseline.epoch) ||
|
|
2020
|
+
!Number.isSafeInteger(raw.factorBaseline.used) || raw.factorBaseline.used < 0 ||
|
|
2021
|
+
finiteTimestamp(raw.factorBaseline.observedAt, nowMs) === undefined ||
|
|
2022
|
+
raw.factorBaseline.observedAt > nowMs ||
|
|
2023
|
+
!Number.isSafeInteger(raw.knownLocalUsed) || raw.knownLocalUsed < 0 ||
|
|
1711
2024
|
!validGovernorEpoch(raw.epoch)
|
|
1712
2025
|
) return null;
|
|
1713
2026
|
return {
|
|
@@ -1718,6 +2031,9 @@ function normalizeGovernorBudget(raw, nowMs) {
|
|
|
1718
2031
|
roundRobinCursor: raw.roundRobinCursor,
|
|
1719
2032
|
lastExternalFactor: raw.lastExternalFactor,
|
|
1720
2033
|
epoch: raw.epoch,
|
|
2034
|
+
source: raw.source,
|
|
2035
|
+
factorBaseline: { ...raw.factorBaseline },
|
|
2036
|
+
knownLocalUsed: raw.knownLocalUsed,
|
|
1721
2037
|
};
|
|
1722
2038
|
}
|
|
1723
2039
|
|
|
@@ -1756,18 +2072,19 @@ function normalizeGovernorIntent(raw, nowMs) {
|
|
|
1756
2072
|
|
|
1757
2073
|
function normalizeGovernorReservation(raw, nowMs) {
|
|
1758
2074
|
if (!exactKeys(raw, [
|
|
1759
|
-
"leaseId", "intentId", "costs", "actualCosts", "notBefore", "status", "epochs",
|
|
2075
|
+
"leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
|
|
1760
2076
|
"startedAt", "completedAt", "outcome",
|
|
1761
2077
|
])) return null;
|
|
1762
2078
|
const costs = exactResourceCosts(raw.costs);
|
|
1763
2079
|
const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
|
|
2080
|
+
const accountedCosts = exactResourceCosts(raw.accountedCosts);
|
|
1764
2081
|
const epochs = exactKeys(raw.epochs, RATE_RESOURCES) &&
|
|
1765
2082
|
RATE_RESOURCES.every((resource) => raw.epochs[resource] === null || validGovernorEpoch(raw.epochs[resource]))
|
|
1766
2083
|
? { ...raw.epochs }
|
|
1767
2084
|
: null;
|
|
1768
2085
|
if (
|
|
1769
2086
|
!validGovernorId(raw.leaseId) || !validGovernorId(raw.intentId) ||
|
|
1770
|
-
!costs || actualCosts === null && raw.actualCosts !== null || !epochs ||
|
|
2087
|
+
!costs || actualCosts === null && raw.actualCosts !== null || !accountedCosts || !epochs ||
|
|
1771
2088
|
finiteTimestamp(raw.notBefore, nowMs) === undefined ||
|
|
1772
2089
|
!GOVERNOR_RESERVATION_STATUSES.has(raw.status) ||
|
|
1773
2090
|
finiteTimestamp(raw.startedAt, nowMs, { nullable: true }) === undefined ||
|
|
@@ -1777,23 +2094,42 @@ function normalizeGovernorReservation(raw, nowMs) {
|
|
|
1777
2094
|
(raw.outcome !== null && !GOVERNOR_OUTCOMES.has(raw.outcome))
|
|
1778
2095
|
) return null;
|
|
1779
2096
|
if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
|
|
1780
|
-
|
|
2097
|
+
if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
|
|
2098
|
+
return { ...raw, costs, actualCosts, accountedCosts, epochs };
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
function normalizeCoreObserver(raw, nowMs) {
|
|
2102
|
+
if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
|
|
2103
|
+
if (
|
|
2104
|
+
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
|
|
2105
|
+
!GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
|
|
2106
|
+
finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
|
|
2107
|
+
finiteTimestamp(raw.nextAt, nowMs) === undefined
|
|
2108
|
+
) return null;
|
|
2109
|
+
return { ...raw };
|
|
1781
2110
|
}
|
|
1782
2111
|
|
|
1783
2112
|
function normalizeProbeClaim(raw, nowMs) {
|
|
1784
2113
|
if (raw === null) return null;
|
|
1785
|
-
if (!exactKeys(raw, [
|
|
2114
|
+
if (!exactKeys(raw, [
|
|
2115
|
+
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds", "resources",
|
|
2116
|
+
])) {
|
|
1786
2117
|
return undefined;
|
|
1787
2118
|
}
|
|
2119
|
+
const resources = Array.isArray(raw.resources) && raw.resources.length > 0 &&
|
|
2120
|
+
raw.resources.every((resource) => RATE_RESOURCES.includes(resource))
|
|
2121
|
+
? [...new Set(raw.resources)]
|
|
2122
|
+
: null;
|
|
1788
2123
|
if (
|
|
1789
2124
|
!validGovernorId(raw.ownerLeaseId) || !validGovernorId(raw.nonce) ||
|
|
1790
2125
|
finiteTimestamp(raw.leaseUntil, nowMs) === undefined ||
|
|
1791
2126
|
finiteTimestamp(raw.nextAt, nowMs) === undefined ||
|
|
1792
2127
|
finiteTimestamp(raw.claimAt, nowMs) === undefined || raw.claimAt > nowMs ||
|
|
1793
2128
|
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
1794
|
-
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
2129
|
+
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12))) ||
|
|
2130
|
+
!resources
|
|
1795
2131
|
) return undefined;
|
|
1796
|
-
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
2132
|
+
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)], resources };
|
|
1797
2133
|
}
|
|
1798
2134
|
|
|
1799
2135
|
function normalizeManualProbe(raw, nowMs) {
|
|
@@ -1810,13 +2146,14 @@ function normalizeManualProbe(raw, nowMs) {
|
|
|
1810
2146
|
|
|
1811
2147
|
function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
1812
2148
|
if (!exactKeys(raw, [
|
|
1813
|
-
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
2149
|
+
"version", "epochs", "budgets", "observers", "probeClaim", "probeOutcome", "leases",
|
|
1814
2150
|
"intents", "reservations", "manualProbe",
|
|
1815
2151
|
]) || raw.version !== GOVERNOR_STATE_VERSION) return null;
|
|
1816
2152
|
if (
|
|
1817
2153
|
!exactKeys(raw.epochs, RATE_RESOURCES) ||
|
|
1818
2154
|
RATE_RESOURCES.some((resource) => raw.epochs[resource] !== null && !validGovernorEpoch(raw.epochs[resource])) ||
|
|
1819
2155
|
!isRecord(raw.budgets) || Object.keys(raw.budgets).some((resource) => !RATE_RESOURCES.includes(resource)) ||
|
|
2156
|
+
!exactKeys(raw.observers, ["core"]) ||
|
|
1820
2157
|
!isRecord(raw.leases) || !isRecord(raw.intents) || !isRecord(raw.reservations) ||
|
|
1821
2158
|
!exactKeys(raw.probeOutcome, ["status", "at", "nextAt"]) ||
|
|
1822
2159
|
!GOVERNOR_PROBE_STATUSES.has(raw.probeOutcome.status) ||
|
|
@@ -1828,8 +2165,10 @@ function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
|
1828
2165
|
const state = emptyGovernorState();
|
|
1829
2166
|
state.epochs = { ...raw.epochs };
|
|
1830
2167
|
state.probeOutcome = { ...raw.probeOutcome };
|
|
2168
|
+
state.observers.core = normalizeCoreObserver(raw.observers.core, nowMs);
|
|
2169
|
+
if (!state.observers.core) return null;
|
|
1831
2170
|
for (const resource of Object.keys(raw.budgets)) {
|
|
1832
|
-
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs);
|
|
2171
|
+
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs, resource);
|
|
1833
2172
|
if (!budget) return null;
|
|
1834
2173
|
state.budgets[resource] = budget;
|
|
1835
2174
|
}
|
|
@@ -1869,7 +2208,36 @@ function serializeGovernorState(state) {
|
|
|
1869
2208
|
return `${JSON.stringify(state)}\n`;
|
|
1870
2209
|
}
|
|
1871
2210
|
|
|
1872
|
-
function
|
|
2211
|
+
function migrateGovernorState(raw, nowMs) {
|
|
2212
|
+
if (!exactKeys(raw, [
|
|
2213
|
+
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
2214
|
+
"intents", "reservations", "manualProbe",
|
|
2215
|
+
]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
|
|
2216
|
+
const migrated = structuredClone(raw);
|
|
2217
|
+
migrated.version = GOVERNOR_STATE_VERSION;
|
|
2218
|
+
migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
|
|
2219
|
+
// The v1 core value came from /rate_limit, which is not authoritative. Do not
|
|
2220
|
+
// preserve it as bootstrap evidence. GraphQL remains valid.
|
|
2221
|
+
delete migrated.budgets.core;
|
|
2222
|
+
migrated.epochs.core = null;
|
|
2223
|
+
if (migrated.budgets.graphql) {
|
|
2224
|
+
const budget = migrated.budgets.graphql;
|
|
2225
|
+
budget.source = "rate-limit-probe";
|
|
2226
|
+
budget.factorBaseline = {
|
|
2227
|
+
epoch: budget.epoch,
|
|
2228
|
+
used: budget.used,
|
|
2229
|
+
observedAt: budget.observedAt,
|
|
2230
|
+
};
|
|
2231
|
+
budget.knownLocalUsed = 0;
|
|
2232
|
+
}
|
|
2233
|
+
for (const reservation of Object.values(migrated.reservations)) {
|
|
2234
|
+
reservation.accountedCosts = { core: 0, graphql: 0 };
|
|
2235
|
+
}
|
|
2236
|
+
if (migrated.probeClaim) migrated.probeClaim.resources = [...RATE_RESOURCES];
|
|
2237
|
+
return normalizeGovernorState(migrated, nowMs);
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
1873
2241
|
let raw;
|
|
1874
2242
|
try {
|
|
1875
2243
|
raw = readFileSync(path, "utf8");
|
|
@@ -1879,8 +2247,16 @@ function readGovernorState(path, nowMs) {
|
|
|
1879
2247
|
: { ok: false, reason: "unwritable" };
|
|
1880
2248
|
}
|
|
1881
2249
|
try {
|
|
1882
|
-
const
|
|
1883
|
-
|
|
2250
|
+
const parsed = JSON.parse(raw);
|
|
2251
|
+
const normalized = normalizeGovernorState(parsed, nowMs);
|
|
2252
|
+
if (normalized) return { ok: true, value: normalized };
|
|
2253
|
+
const migrated = migrateGovernorState(parsed, nowMs);
|
|
2254
|
+
if (!migrated) return { ok: false, reason: "corrupt" };
|
|
2255
|
+
if (!persistMigration) return { ok: true, value: migrated, migrated: true };
|
|
2256
|
+
// Read-only callers persist the exact migration while they still hold the
|
|
2257
|
+
// scope lock. Mutating callers use this same value for their final write.
|
|
2258
|
+
const written = writeGovernorState(path, migrated);
|
|
2259
|
+
return written.ok ? { ok: true, value: migrated, migrated: true } : written;
|
|
1884
2260
|
} catch {
|
|
1885
2261
|
return { ok: false, reason: "corrupt" };
|
|
1886
2262
|
}
|
|
@@ -2109,13 +2485,21 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
2109
2485
|
// held, immediately before it reads and mutates the shared state.
|
|
2110
2486
|
const current = currentGovernorScope(scope);
|
|
2111
2487
|
if (!current.ok) return current;
|
|
2112
|
-
const loaded = readGovernorState(scope.path, at);
|
|
2488
|
+
const loaded = readGovernorState(scope.path, at, { persistMigration: false });
|
|
2113
2489
|
if (!loaded.ok) return loaded;
|
|
2114
2490
|
const state = loaded.value;
|
|
2491
|
+
// A mutator can reject without writing. Keep an untouched migration so
|
|
2492
|
+
// that even that path upgrades the file once before releasing the lock.
|
|
2493
|
+
const migration = loaded.migrated ? structuredClone(state) : null;
|
|
2494
|
+
const returnAfterMigration = (result) => {
|
|
2495
|
+
if (migration === null) return result;
|
|
2496
|
+
const written = writeGovernorState(scope.path, migration);
|
|
2497
|
+
return written.ok ? result : written;
|
|
2498
|
+
};
|
|
2115
2499
|
const value = mutate(state, at);
|
|
2116
|
-
if (value?.ok === false && value.write !== true) return value;
|
|
2500
|
+
if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
|
|
2117
2501
|
const normalized = normalizeGovernorState(state, at);
|
|
2118
|
-
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
2502
|
+
if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
|
|
2119
2503
|
const written = writeGovernorState(scope.path, normalized);
|
|
2120
2504
|
if (!written.ok) return written;
|
|
2121
2505
|
return value?.ok === false ? { ok: false, reason: value.reason } : { ok: true, value: value?.value };
|
|
@@ -2168,6 +2552,7 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
2168
2552
|
intentId: grant.intentId,
|
|
2169
2553
|
costs: grant.costs,
|
|
2170
2554
|
actualCosts: null,
|
|
2555
|
+
accountedCosts: { core: 0, graphql: 0 },
|
|
2171
2556
|
notBefore: grant.notBefore,
|
|
2172
2557
|
status: "scheduled",
|
|
2173
2558
|
epochs: { core: grant.epochs.core ?? null, graphql: grant.epochs.graphql ?? null },
|
|
@@ -2246,11 +2631,32 @@ function claimProbe(scope, leaseId, nowMs) {
|
|
|
2246
2631
|
if (state.probeClaim && state.probeClaim.leaseUntil > at) {
|
|
2247
2632
|
return { value: { status: "waiting", leaseUntil: state.probeClaim.leaseUntil } };
|
|
2248
2633
|
}
|
|
2249
|
-
const
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
const
|
|
2634
|
+
const coreResetAt = Number.isFinite(state.budgets.core?.resetMs)
|
|
2635
|
+
? state.budgets.core.resetMs + BUDGET_RESET_GRACE_MS
|
|
2636
|
+
: Number.POSITIVE_INFINITY;
|
|
2637
|
+
const graphqlResetAt = Number.isFinite(state.budgets.graphql?.resetMs)
|
|
2638
|
+
? state.budgets.graphql.resetMs + BUDGET_RESET_GRACE_MS
|
|
2639
|
+
: Number.POSITIVE_INFINITY;
|
|
2640
|
+
const dueAt = {
|
|
2641
|
+
core: Math.min(
|
|
2642
|
+
state.observers.core.nextAt,
|
|
2643
|
+
coreResetAt,
|
|
2644
|
+
),
|
|
2645
|
+
graphql: Math.min(
|
|
2646
|
+
state.probeOutcome.nextAt,
|
|
2647
|
+
graphqlResetAt,
|
|
2648
|
+
// A core reset starts a new shared accounting epoch. Refresh the free
|
|
2649
|
+
// GraphQL counter in that same nonce so external-spend baselines and
|
|
2650
|
+
// the protected one-publication reset contract advance together.
|
|
2651
|
+
coreResetAt,
|
|
2652
|
+
),
|
|
2653
|
+
};
|
|
2654
|
+
const nextAt = Math.min(...Object.values(dueAt));
|
|
2253
2655
|
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
2656
|
+
// One nonce owns the control-plane transition, but only the sources whose
|
|
2657
|
+
// persisted clocks are due may be read. In particular, a GraphQL minute
|
|
2658
|
+
// sample cannot spend or retry the held core observer before its reset.
|
|
2659
|
+
const resources = RATE_RESOURCES.filter((resource) => dueAt[resource] <= at);
|
|
2254
2660
|
const nonce = randomUUID();
|
|
2255
2661
|
const startedReservationIds = Object.entries(state.reservations)
|
|
2256
2662
|
.filter(([, reservation]) => reservation.status === "started")
|
|
@@ -2262,9 +2668,24 @@ function claimProbe(scope, leaseId, nowMs) {
|
|
|
2262
2668
|
nextAt: at,
|
|
2263
2669
|
claimAt: at,
|
|
2264
2670
|
startedReservationIds,
|
|
2671
|
+
resources,
|
|
2265
2672
|
};
|
|
2266
|
-
|
|
2267
|
-
|
|
2673
|
+
if (resources.includes("core")) {
|
|
2674
|
+
state.observers.core.outcome = "waiting";
|
|
2675
|
+
state.observers.core.at = at;
|
|
2676
|
+
state.observers.core.nextAt = at + GOVERNOR_PROBE_LEASE_MS;
|
|
2677
|
+
}
|
|
2678
|
+
if (resources.includes("graphql")) {
|
|
2679
|
+
state.probeOutcome = { status: "waiting", at, nextAt: at + GOVERNOR_PROBE_LEASE_MS };
|
|
2680
|
+
}
|
|
2681
|
+
return { value: {
|
|
2682
|
+
status: "claimed",
|
|
2683
|
+
nonce,
|
|
2684
|
+
leaseUntil: state.probeClaim.leaseUntil,
|
|
2685
|
+
startedReservationIds,
|
|
2686
|
+
resources,
|
|
2687
|
+
coreEtag: state.observers.core.etag,
|
|
2688
|
+
} };
|
|
2268
2689
|
});
|
|
2269
2690
|
}
|
|
2270
2691
|
|
|
@@ -2279,18 +2700,41 @@ function renewProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
|
2279
2700
|
});
|
|
2280
2701
|
}
|
|
2281
2702
|
|
|
2282
|
-
function
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2703
|
+
function budgetFromObservation(raw, previous, nowMs, {
|
|
2704
|
+
resource,
|
|
2705
|
+
source,
|
|
2706
|
+
receivedAt = nowMs,
|
|
2707
|
+
clearProvenBlock = false,
|
|
2708
|
+
allowEpochChange = false,
|
|
2709
|
+
} = {}) {
|
|
2710
|
+
const normalized = normalizeBudgetResource({ ...raw, observedAt: receivedAt });
|
|
2711
|
+
if (
|
|
2712
|
+
!normalized || normalized.resetMs <= nowMs || receivedAt > nowMs ||
|
|
2713
|
+
!RATE_RESOURCES.includes(resource) ||
|
|
2714
|
+
!["response-header", "core-observer", "rate-limit-probe"].includes(source) ||
|
|
2715
|
+
resource === "core" && source === "rate-limit-probe"
|
|
2716
|
+
) return { status: "invalid" };
|
|
2717
|
+
const epoch = `${normalized.limit}:${normalized.resetMs}`;
|
|
2718
|
+
// Endpoint responses can carry cached or endpoint-specific rate headers.
|
|
2719
|
+
// They may refine the counter inside the epoch established by its claimed
|
|
2720
|
+
// observer, but only that observer may move the resource into a new epoch.
|
|
2721
|
+
if (!allowEpochChange && (!previous || previous.epoch !== epoch)) {
|
|
2722
|
+
return { status: "ignored", epoch };
|
|
2723
|
+
}
|
|
2724
|
+
if (previous) {
|
|
2725
|
+
if (normalized.resetMs < previous.resetMs) return { status: "ignored" };
|
|
2726
|
+
if (normalized.resetMs === previous.resetMs && normalized.used < previous.used) {
|
|
2727
|
+
return { status: "ignored" };
|
|
2728
|
+
}
|
|
2729
|
+
if (
|
|
2730
|
+
normalized.resetMs === previous.resetMs && normalized.used === previous.used &&
|
|
2731
|
+
receivedAt <= previous.observedAt
|
|
2732
|
+
) return { status: "ignored" };
|
|
2733
|
+
}
|
|
2291
2734
|
const epochChanged = !previous || previous.epoch !== epoch;
|
|
2292
|
-
const
|
|
2293
|
-
|
|
2735
|
+
const preserveBlock = !clearProvenBlock || !epochChanged;
|
|
2736
|
+
const blockActive = preserveBlock && Number.isFinite(previous?.blockUntil) && previous.blockUntil > nowMs;
|
|
2737
|
+
const next = {
|
|
2294
2738
|
...normalized,
|
|
2295
2739
|
blockUntil: blockActive ? previous.blockUntil : null,
|
|
2296
2740
|
blockReason: blockActive ? previous.blockReason : null,
|
|
@@ -2298,7 +2742,13 @@ function budgetFromProbe(raw, previous, nowMs) {
|
|
|
2298
2742
|
roundRobinCursor: previous?.roundRobinCursor ?? null,
|
|
2299
2743
|
lastExternalFactor: previous?.lastExternalFactor ?? 1,
|
|
2300
2744
|
epoch,
|
|
2745
|
+
source,
|
|
2746
|
+
factorBaseline: epochChanged || !previous?.factorBaseline
|
|
2747
|
+
? { epoch, used: normalized.used, observedAt: receivedAt }
|
|
2748
|
+
: { ...previous.factorBaseline },
|
|
2749
|
+
knownLocalUsed: epochChanged ? 0 : previous.knownLocalUsed,
|
|
2301
2750
|
};
|
|
2751
|
+
return { status: "accepted", budget: next, epochChanged };
|
|
2302
2752
|
}
|
|
2303
2753
|
|
|
2304
2754
|
function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
@@ -2307,45 +2757,103 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2307
2757
|
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2308
2758
|
return { ok: false, reason: "stale" };
|
|
2309
2759
|
}
|
|
2310
|
-
const nextBudgets = {};
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2760
|
+
const nextBudgets = { ...state.budgets };
|
|
2761
|
+
const nextEpochs = { ...state.epochs };
|
|
2762
|
+
const changedResources = [];
|
|
2763
|
+
const resetResources = [];
|
|
2764
|
+
if (!claim.resources.every((resource) => budgets?.[resource])) {
|
|
2765
|
+
return { ok: false, reason: "corrupt" };
|
|
2314
2766
|
}
|
|
2315
|
-
const
|
|
2316
|
-
resource
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2767
|
+
for (const resource of claim.resources) {
|
|
2768
|
+
const supplied = budgets?.[resource];
|
|
2769
|
+
if (!supplied) continue;
|
|
2770
|
+
const raw = supplied.budget ?? supplied;
|
|
2771
|
+
const source = supplied.source ?? (resource === "core" ? "core-observer" : "rate-limit-probe");
|
|
2772
|
+
const ownerSource = resource === "core" ? "core-observer" : "rate-limit-probe";
|
|
2773
|
+
if (source !== ownerSource) return { ok: false, reason: "corrupt" };
|
|
2774
|
+
const observed = budgetFromObservation(raw, state.budgets[resource], at, {
|
|
2775
|
+
resource,
|
|
2776
|
+
source,
|
|
2777
|
+
receivedAt: supplied.receivedAt ?? at,
|
|
2778
|
+
clearProvenBlock: true,
|
|
2779
|
+
allowEpochChange: true,
|
|
2780
|
+
});
|
|
2781
|
+
if (observed.status === "invalid") return { ok: false, reason: "corrupt" };
|
|
2782
|
+
if (observed.status === "ignored") continue;
|
|
2783
|
+
nextBudgets[resource] = observed.budget;
|
|
2784
|
+
if (resource === "core" && supplied.blocked === true) {
|
|
2785
|
+
nextBudgets[resource].blockUntil = observed.budget.resetMs;
|
|
2786
|
+
nextBudgets[resource].blockReason = "rate-limit";
|
|
2787
|
+
}
|
|
2788
|
+
nextEpochs[resource] = observed.budget.epoch;
|
|
2789
|
+
changedResources.push(resource);
|
|
2790
|
+
if (observed.epochChanged && state.epochs[resource] !== null) resetResources.push(resource);
|
|
2791
|
+
}
|
|
2792
|
+
if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
|
|
2793
|
+
const completedBeforeClaim = Object.entries(state.reservations)
|
|
2794
|
+
.filter(([, reservation]) =>
|
|
2795
|
+
reservation.status === "completed" && reservation.completedAt < claim.claimAt);
|
|
2796
|
+
for (const resource of changedResources) {
|
|
2320
2797
|
const previous = state.budgets[resource];
|
|
2321
|
-
if (!previous || nextBudgets[resource].
|
|
2322
|
-
const
|
|
2323
|
-
|
|
2324
|
-
|
|
2798
|
+
if (!previous || nextBudgets[resource].epoch !== previous.epoch) continue;
|
|
2799
|
+
const baseline = previous.factorBaseline;
|
|
2800
|
+
if (!baseline || baseline.epoch !== nextBudgets[resource].epoch || nextBudgets[resource].used < baseline.used) {
|
|
2801
|
+
continue;
|
|
2802
|
+
}
|
|
2803
|
+
const legacyCompletedDelta = completedBeforeClaim
|
|
2804
|
+
.reduce((total, [, reservation]) =>
|
|
2805
|
+
total + (reservation.completedAt > baseline.observedAt
|
|
2806
|
+
? reservationCost(reservation, resource, state.leases, at)
|
|
2807
|
+
: 0), 0);
|
|
2325
2808
|
const factor = nextExternalFactor({
|
|
2326
2809
|
lastExternalFactor: previous.lastExternalFactor,
|
|
2327
|
-
globalUsedDelta: nextBudgets[resource].used -
|
|
2328
|
-
sharedCompletedDelta,
|
|
2810
|
+
globalUsedDelta: nextBudgets[resource].used - baseline.used,
|
|
2811
|
+
sharedCompletedDelta: previous.knownLocalUsed + legacyCompletedDelta,
|
|
2329
2812
|
});
|
|
2330
2813
|
if (factor === null) return { ok: false, reason: "corrupt" };
|
|
2331
2814
|
nextBudgets[resource].lastExternalFactor = factor;
|
|
2815
|
+
nextBudgets[resource].factorBaseline = {
|
|
2816
|
+
epoch: nextBudgets[resource].epoch,
|
|
2817
|
+
used: nextBudgets[resource].used,
|
|
2818
|
+
observedAt: nextBudgets[resource].observedAt,
|
|
2819
|
+
};
|
|
2820
|
+
nextBudgets[resource].knownLocalUsed = 0;
|
|
2332
2821
|
}
|
|
2333
|
-
const
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
if (
|
|
2338
|
-
delete state.reservations[id];
|
|
2339
|
-
}
|
|
2822
|
+
for (const [id, reservation] of completedBeforeClaim) {
|
|
2823
|
+
const accountedByEveryResource = RATE_RESOURCES.every((resource) =>
|
|
2824
|
+
reservationCost(reservation, resource, state.leases, at) === 0 ||
|
|
2825
|
+
nextBudgets[resource]?.factorBaseline?.observedAt >= reservation.completedAt);
|
|
2826
|
+
if (accountedByEveryResource) delete state.reservations[id];
|
|
2340
2827
|
}
|
|
2341
2828
|
state.budgets = nextBudgets;
|
|
2342
2829
|
state.epochs = nextEpochs;
|
|
2343
2830
|
state.probeClaim = null;
|
|
2344
|
-
|
|
2345
|
-
|
|
2831
|
+
if (claim.resources.includes("core")) {
|
|
2832
|
+
const core = nextBudgets.core;
|
|
2833
|
+
state.observers.core = {
|
|
2834
|
+
etag: typeof budgets.core.etag === "string" ? budgets.core.etag : state.observers.core.etag,
|
|
2835
|
+
outcome: "healthy",
|
|
2836
|
+
at,
|
|
2837
|
+
nextAt: core.remaining === 0
|
|
2838
|
+
? core.resetMs + BUDGET_RESET_GRACE_MS
|
|
2839
|
+
: at + BUDGET_PROBE_MS,
|
|
2840
|
+
};
|
|
2841
|
+
}
|
|
2842
|
+
if (claim.resources.includes("graphql")) {
|
|
2843
|
+
state.probeOutcome = {
|
|
2844
|
+
status: "healthy",
|
|
2845
|
+
at,
|
|
2846
|
+
nextAt: at + BUDGET_PROBE_MS,
|
|
2847
|
+
};
|
|
2848
|
+
}
|
|
2849
|
+
if (resetResources.length > 0) state.manualProbe = null;
|
|
2346
2850
|
else if (state.manualProbe &&
|
|
2347
2851
|
Object.values(nextEpochs).includes(state.manualProbe.requestedEpoch) &&
|
|
2348
|
-
RATE_RESOURCES.some((resource) => availableForGrant({
|
|
2852
|
+
RATE_RESOURCES.some((resource) => availableForGrant({
|
|
2853
|
+
budget: nextBudgets[resource],
|
|
2854
|
+
resource,
|
|
2855
|
+
nowMs: at,
|
|
2856
|
+
}).spendable === 0)) {
|
|
2349
2857
|
state.manualProbe.satisfiedAt = at;
|
|
2350
2858
|
}
|
|
2351
2859
|
scheduleGovernorState(state, at);
|
|
@@ -2360,8 +2868,18 @@ function failProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
|
2360
2868
|
return { ok: false, reason: "stale" };
|
|
2361
2869
|
}
|
|
2362
2870
|
state.probeClaim = null;
|
|
2363
|
-
|
|
2364
|
-
|
|
2871
|
+
if (claim.resources.includes("graphql")) {
|
|
2872
|
+
state.probeOutcome = { status: "failed", at, nextAt: at + BUDGET_PROBE_MS };
|
|
2873
|
+
}
|
|
2874
|
+
if (claim.resources.includes("core")) {
|
|
2875
|
+
state.observers.core = {
|
|
2876
|
+
...state.observers.core,
|
|
2877
|
+
outcome: "failed",
|
|
2878
|
+
at,
|
|
2879
|
+
nextAt: at + BUDGET_PROBE_MS,
|
|
2880
|
+
};
|
|
2881
|
+
}
|
|
2882
|
+
return { value: { retryAt: at + BUDGET_PROBE_MS } };
|
|
2365
2883
|
});
|
|
2366
2884
|
}
|
|
2367
2885
|
|
|
@@ -2374,7 +2892,16 @@ function requestManualProbe(scope, leaseId, epoch, observedAt, nowMs) {
|
|
|
2374
2892
|
return { value: { status: state.manualProbe.satisfiedAt === null ? "pending" : "satisfied", ...state.manualProbe } };
|
|
2375
2893
|
}
|
|
2376
2894
|
state.manualProbe = { requestedEpoch: epoch, baselineObservedAt: observedAt, satisfiedAt: null };
|
|
2377
|
-
|
|
2895
|
+
if (state.epochs.core === epoch) {
|
|
2896
|
+
const core = state.budgets.core;
|
|
2897
|
+
const heldThroughReset = core?.blockUntil > at && core.blockUntil === core.resetMs;
|
|
2898
|
+
state.observers.core.nextAt = heldThroughReset
|
|
2899
|
+
? Math.max(state.observers.core.nextAt, core.blockUntil + BUDGET_RESET_GRACE_MS)
|
|
2900
|
+
: Math.min(state.observers.core.nextAt || at, at);
|
|
2901
|
+
}
|
|
2902
|
+
if (state.epochs.graphql === epoch) {
|
|
2903
|
+
state.probeOutcome.nextAt = Math.min(state.probeOutcome.nextAt || at, at);
|
|
2904
|
+
}
|
|
2378
2905
|
return { value: { status: "pending", ...state.manualProbe } };
|
|
2379
2906
|
});
|
|
2380
2907
|
}
|
|
@@ -2410,9 +2937,15 @@ function registerIntent(scope, intent) {
|
|
|
2410
2937
|
state.intents[intent.id] = normalized;
|
|
2411
2938
|
const scheduled = scheduleGovernorState(state, nowMs);
|
|
2412
2939
|
const reservation = state.reservations[reservationId];
|
|
2940
|
+
const grant = scheduled.grants.find((item) => item.id === reservationId);
|
|
2413
2941
|
const denial = scheduled.denied.find((item) => item.intentId === intent.id);
|
|
2414
2942
|
return { value: reservation
|
|
2415
|
-
? {
|
|
2943
|
+
? {
|
|
2944
|
+
status: "scheduled",
|
|
2945
|
+
reservationId,
|
|
2946
|
+
...reservation,
|
|
2947
|
+
...sharedLaneEvidence(grant),
|
|
2948
|
+
}
|
|
2416
2949
|
: {
|
|
2417
2950
|
status: denial?.mode ?? "pending",
|
|
2418
2951
|
intentId: intent.id,
|
|
@@ -2430,7 +2963,13 @@ function readIntentDecision(scope, intentId, nowMs) {
|
|
|
2430
2963
|
const scheduled = scheduleGovernorState(state, at);
|
|
2431
2964
|
const reservationId = `reservation:${intentId}`;
|
|
2432
2965
|
const reservation = state.reservations[reservationId];
|
|
2433
|
-
|
|
2966
|
+
const grant = scheduled.grants.find((item) => item.id === reservationId);
|
|
2967
|
+
if (reservation) return { value: {
|
|
2968
|
+
status: reservation.status,
|
|
2969
|
+
reservationId,
|
|
2970
|
+
...reservation,
|
|
2971
|
+
...sharedLaneEvidence(grant),
|
|
2972
|
+
} };
|
|
2434
2973
|
if (!state.intents[intentId]) return { ok: false, reason: "stale" };
|
|
2435
2974
|
const denial = scheduled.denied.find((item) => item.intentId === intentId);
|
|
2436
2975
|
return { value: {
|
|
@@ -2476,7 +3015,7 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
2476
3015
|
for (const resource of RATE_RESOURCES.filter((name) => reservation.costs[name] > 0)) {
|
|
2477
3016
|
const budget = state.budgets[resource];
|
|
2478
3017
|
if (
|
|
2479
|
-
!budget || at - budget.observedAt >
|
|
3018
|
+
!budget || at - budget.observedAt > budgetSnapshotTtl(resource) || at >= budget.resetMs ||
|
|
2480
3019
|
state.epochs[resource] !== reservation.epochs[resource] ||
|
|
2481
3020
|
(Number.isFinite(budget.blockUntil) && budget.blockUntil > at)
|
|
2482
3021
|
) {
|
|
@@ -2514,10 +3053,107 @@ function completeReservation(scope, reservationId, completion, nowMs) {
|
|
|
2514
3053
|
reservation.completedAt = at;
|
|
2515
3054
|
reservation.outcome = completion.outcome;
|
|
2516
3055
|
reservation.actualCosts = { ...measured };
|
|
3056
|
+
reservation.accountedCosts = { core: 0, graphql: 0 };
|
|
2517
3057
|
return { value: { status: "completed", actualCosts: reservation.actualCosts } };
|
|
2518
3058
|
});
|
|
2519
3059
|
}
|
|
2520
3060
|
|
|
3061
|
+
function settleReservationWithBudgetObservations(
|
|
3062
|
+
scope,
|
|
3063
|
+
leaseId,
|
|
3064
|
+
reservationId,
|
|
3065
|
+
completion,
|
|
3066
|
+
nowMs,
|
|
3067
|
+
) {
|
|
3068
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
3069
|
+
const reservation = state.reservations[reservationId];
|
|
3070
|
+
const lease = state.leases[leaseId];
|
|
3071
|
+
if (
|
|
3072
|
+
!lease || lease.expiresAt <= at || !reservation ||
|
|
3073
|
+
reservation.leaseId !== leaseId || reservation.status !== "started" ||
|
|
3074
|
+
!GOVERNOR_OUTCOMES.has(completion?.outcome)
|
|
3075
|
+
) return { ok: false, reason: "stale" };
|
|
3076
|
+
const measured = completion.outcome === "measured-success"
|
|
3077
|
+
? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
|
|
3078
|
+
: reservation.costs;
|
|
3079
|
+
if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
|
|
3080
|
+
return { ok: false, reason: "corrupt" };
|
|
3081
|
+
}
|
|
3082
|
+
const observations = Array.isArray(completion.observations) ? completion.observations : [];
|
|
3083
|
+
if (observations.some((observation) =>
|
|
3084
|
+
!isRecord(observation) || observation.source !== "response-header" ||
|
|
3085
|
+
!Number.isFinite(observation.receivedAt) || observation.receivedAt > at)) {
|
|
3086
|
+
return { ok: false, reason: "corrupt" };
|
|
3087
|
+
}
|
|
3088
|
+
const acceptedCosts = { core: 0, graphql: 0 };
|
|
3089
|
+
const acceptedAfterBaseline = { core: 0, graphql: 0 };
|
|
3090
|
+
for (const resource of RATE_RESOURCES) {
|
|
3091
|
+
const candidates = observations
|
|
3092
|
+
.filter((observation) => observation.resource === resource)
|
|
3093
|
+
.sort((left, right) =>
|
|
3094
|
+
left.resetMs - right.resetMs || left.used - right.used || left.receivedAt - right.receivedAt);
|
|
3095
|
+
for (const observation of candidates) {
|
|
3096
|
+
const observed = budgetFromObservation(observation, state.budgets[resource], at, {
|
|
3097
|
+
resource,
|
|
3098
|
+
source: observation.source,
|
|
3099
|
+
receivedAt: observation.receivedAt,
|
|
3100
|
+
clearProvenBlock: false,
|
|
3101
|
+
});
|
|
3102
|
+
if (observed.status === "invalid") continue;
|
|
3103
|
+
const definiteCost = Number.isSafeInteger(observation.cost) && observation.cost >= 0
|
|
3104
|
+
? observation.cost
|
|
3105
|
+
: 0;
|
|
3106
|
+
if (observed.status === "accepted") {
|
|
3107
|
+
state.budgets[resource] = observed.budget;
|
|
3108
|
+
state.epochs[resource] = observed.budget.epoch;
|
|
3109
|
+
acceptedCosts[resource] += definiteCost;
|
|
3110
|
+
if (observation.receivedAt > observed.budget.factorBaseline.observedAt) {
|
|
3111
|
+
acceptedAfterBaseline[resource] += definiteCost;
|
|
3112
|
+
}
|
|
3113
|
+
} else if (
|
|
3114
|
+
observation.source === "response-header" &&
|
|
3115
|
+
state.budgets[resource]?.epoch === `${observation.limit}:${observation.resetMs}` &&
|
|
3116
|
+
state.budgets[resource].used >= observation.used
|
|
3117
|
+
) {
|
|
3118
|
+
// A newer same-window counter already includes this completed call.
|
|
3119
|
+
acceptedCosts[resource] += definiteCost;
|
|
3120
|
+
if (observation.receivedAt > state.budgets[resource].factorBaseline.observedAt) {
|
|
3121
|
+
acceptedAfterBaseline[resource] += definiteCost;
|
|
3122
|
+
}
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
reservation.status = "completed";
|
|
3127
|
+
reservation.completedAt = at;
|
|
3128
|
+
reservation.outcome = completion.outcome;
|
|
3129
|
+
reservation.actualCosts = { ...measured };
|
|
3130
|
+
reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3131
|
+
resource,
|
|
3132
|
+
completion.outcome === "measured-success"
|
|
3133
|
+
? Math.min(measured[resource], acceptedCosts[resource])
|
|
3134
|
+
: 0,
|
|
3135
|
+
]));
|
|
3136
|
+
for (const resource of RATE_RESOURCES) {
|
|
3137
|
+
const budget = state.budgets[resource];
|
|
3138
|
+
if (!budget || completion.outcome !== "measured-success") continue;
|
|
3139
|
+
// accountedCosts removes the portion already present in an authoritative
|
|
3140
|
+
// counter from the reservation residual. knownLocalUsed contains only
|
|
3141
|
+
// that same portion when it arrived after the factor baseline. Their sum
|
|
3142
|
+
// is therefore exactly the measured local cost, never twice that cost.
|
|
3143
|
+
budget.knownLocalUsed += Math.min(
|
|
3144
|
+
reservation.accountedCosts[resource],
|
|
3145
|
+
acceptedAfterBaseline[resource],
|
|
3146
|
+
);
|
|
3147
|
+
}
|
|
3148
|
+
scheduleGovernorState(state, at);
|
|
3149
|
+
return { value: {
|
|
3150
|
+
status: "completed",
|
|
3151
|
+
actualCosts: reservation.actualCosts,
|
|
3152
|
+
accountedCosts: reservation.accountedCosts,
|
|
3153
|
+
} };
|
|
3154
|
+
});
|
|
3155
|
+
}
|
|
3156
|
+
|
|
2521
3157
|
function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
2522
3158
|
return mutateGovernor(scope, undefined, (state, nowMs) => {
|
|
2523
3159
|
const budget = state.budgets[resource];
|
|
@@ -2525,6 +3161,17 @@ function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
|
2525
3161
|
!GOVERNOR_BLOCK_REASONS.has(reason)) return { ok: false, reason: "corrupt" };
|
|
2526
3162
|
budget.blockUntil = resetMs;
|
|
2527
3163
|
budget.blockReason = reason;
|
|
3164
|
+
if (resource === "core") {
|
|
3165
|
+
state.observers.core.nextAt = Math.min(
|
|
3166
|
+
state.observers.core.nextAt || resetMs + BUDGET_RESET_GRACE_MS,
|
|
3167
|
+
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3168
|
+
);
|
|
3169
|
+
} else {
|
|
3170
|
+
state.probeOutcome.nextAt = Math.min(
|
|
3171
|
+
state.probeOutcome.nextAt || resetMs + BUDGET_RESET_GRACE_MS,
|
|
3172
|
+
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3173
|
+
);
|
|
3174
|
+
}
|
|
2528
3175
|
return { value: { resource, resetMs, reason } };
|
|
2529
3176
|
});
|
|
2530
3177
|
}
|
|
@@ -2558,7 +3205,8 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
2558
3205
|
const state = result.value;
|
|
2559
3206
|
let status = "healthy";
|
|
2560
3207
|
if (state.probeClaim?.leaseUntil > nowMs) status = "waiting for probe";
|
|
2561
|
-
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] ||
|
|
3208
|
+
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] ||
|
|
3209
|
+
nowMs - state.budgets[resource].observedAt > budgetSnapshotTtl(resource))) status = "stale";
|
|
2562
3210
|
else if (RATE_RESOURCES.some((resource) => state.budgets[resource].blockUntil > nowMs)) status = "blocked";
|
|
2563
3211
|
return {
|
|
2564
3212
|
status,
|
|
@@ -2569,6 +3217,7 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
2569
3217
|
reserve: resourceReserve(budget.limit),
|
|
2570
3218
|
remaining: budget.remaining,
|
|
2571
3219
|
resetMs: budget.resetMs,
|
|
3220
|
+
source: budget.source,
|
|
2572
3221
|
}]] : [];
|
|
2573
3222
|
})),
|
|
2574
3223
|
};
|
|
@@ -2584,7 +3233,7 @@ async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
|
2584
3233
|
}
|
|
2585
3234
|
|
|
2586
3235
|
async function refreshSharedBudget(scope, leaseId, signal, {
|
|
2587
|
-
readBudgets =
|
|
3236
|
+
readBudgets = readSharedBudgetSources,
|
|
2588
3237
|
now = () => scopeNow(scope),
|
|
2589
3238
|
wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
2590
3239
|
inspect = inspectGovernor,
|
|
@@ -2608,7 +3257,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2608
3257
|
if (
|
|
2609
3258
|
RATE_RESOURCES.every((resource) =>
|
|
2610
3259
|
snapshot.value.budgets[resource] &&
|
|
2611
|
-
now() - snapshot.value.budgets[resource].observedAt <=
|
|
3260
|
+
now() - snapshot.value.budgets[resource].observedAt <= budgetSnapshotTtl(resource),
|
|
2612
3261
|
) && snapshot.value.probeOutcome.status === "healthy"
|
|
2613
3262
|
) {
|
|
2614
3263
|
return { ok: true, value: { status: "published", budgets: snapshot.value.budgets } };
|
|
@@ -2619,7 +3268,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2619
3268
|
claim = claimProbe(scope, leaseId, now());
|
|
2620
3269
|
if (!claim.ok || claim.value.status !== "claimed") return claim;
|
|
2621
3270
|
}
|
|
2622
|
-
const { nonce, startedReservationIds } = claim.value;
|
|
3271
|
+
const { nonce, startedReservationIds, resources = RATE_RESOURCES, coreEtag = null } = claim.value;
|
|
2623
3272
|
const drainUntil = now() + GOVERNOR_PROBE_DRAIN_MS;
|
|
2624
3273
|
while (startedReservationIds.length > 0 && now() < drainUntil) {
|
|
2625
3274
|
const snapshot = inspect(scope, now());
|
|
@@ -2650,7 +3299,24 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
2650
3299
|
}
|
|
2651
3300
|
let budgets;
|
|
2652
3301
|
try {
|
|
2653
|
-
budgets = await readBudgets(signal, scope.host
|
|
3302
|
+
budgets = await readBudgets(signal, scope.host, {
|
|
3303
|
+
resources,
|
|
3304
|
+
coreEtag,
|
|
3305
|
+
renewClaim: async () => {
|
|
3306
|
+
const startedAt = now();
|
|
3307
|
+
const extended = await retryGovernorMutation(
|
|
3308
|
+
() => renew(scope, leaseId, nonce, now()),
|
|
3309
|
+
{
|
|
3310
|
+
now,
|
|
3311
|
+
wait,
|
|
3312
|
+
deadline: Math.min(renewed.value.leaseUntil, startedAt + GOVERNOR_PROBE_TRANSITION_MS),
|
|
3313
|
+
signal,
|
|
3314
|
+
},
|
|
3315
|
+
);
|
|
3316
|
+
if (extended.ok) renewed.value.leaseUntil = extended.value.leaseUntil;
|
|
3317
|
+
return extended.ok;
|
|
3318
|
+
},
|
|
3319
|
+
});
|
|
2654
3320
|
} catch {
|
|
2655
3321
|
budgets = null;
|
|
2656
3322
|
}
|
|
@@ -2703,7 +3369,7 @@ function resourcePerTick(table, activeKey) {
|
|
|
2703
3369
|
}
|
|
2704
3370
|
|
|
2705
3371
|
function alertArgs(source, path = source.path) {
|
|
2706
|
-
return [
|
|
3372
|
+
return [apiPath(path), ...apiHostArgs(), "--jq", source.jq];
|
|
2707
3373
|
}
|
|
2708
3374
|
|
|
2709
3375
|
function alertRequestArgs(source) {
|
|
@@ -2778,7 +3444,11 @@ function clearForcedBackoffAfterStart(key, force, status, clear = clearBackoff)
|
|
|
2778
3444
|
return true;
|
|
2779
3445
|
}
|
|
2780
3446
|
|
|
2781
|
-
async function fetchAlertSource(source, signal, now
|
|
3447
|
+
async function fetchAlertSource(source, signal, now, {
|
|
3448
|
+
entities,
|
|
3449
|
+
force = false,
|
|
3450
|
+
request = fetchConditionalEntity,
|
|
3451
|
+
} = {}) {
|
|
2782
3452
|
if (backoffActive(source.key, now)) {
|
|
2783
3453
|
const { note, verdict } = alertBackoff.get(source.key);
|
|
2784
3454
|
// The verdict is replayed alongside the note. Replaying only the note left
|
|
@@ -2790,27 +3460,51 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2790
3460
|
raw: `backoff:${note}`,
|
|
2791
3461
|
completedCalls: 0,
|
|
2792
3462
|
verdict,
|
|
3463
|
+
allNotModified: false,
|
|
3464
|
+
stagedEntities: new Map(),
|
|
2793
3465
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
2794
3466
|
};
|
|
2795
3467
|
}
|
|
2796
3468
|
let completedCalls = 0;
|
|
3469
|
+
const observations = [];
|
|
2797
3470
|
try {
|
|
2798
3471
|
const requests = alertRequestArgs(source);
|
|
2799
3472
|
const payloads = [];
|
|
2800
3473
|
const groups = [];
|
|
3474
|
+
const responses = [];
|
|
2801
3475
|
for (const [index, args] of requests.entries()) {
|
|
2802
3476
|
if (index > 0 && !shouldFetchAlertPriorityLanes(groups[0]?.length ?? 0)) break;
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
3477
|
+
let response = null;
|
|
3478
|
+
try {
|
|
3479
|
+
response = await request({
|
|
3480
|
+
tab: "security",
|
|
3481
|
+
args,
|
|
3482
|
+
operation: "tab:security-endpoint",
|
|
3483
|
+
signal,
|
|
3484
|
+
force,
|
|
3485
|
+
entities,
|
|
3486
|
+
});
|
|
3487
|
+
if (response.status === 200) completedCalls += 1;
|
|
3488
|
+
observations.push(...(response.observations ?? []));
|
|
3489
|
+
responses.push(response);
|
|
3490
|
+
payloads.push(response.body ?? "");
|
|
3491
|
+
groups.push(parseJsonOutput(response.body ?? "").filter((alert) => alert.state === "open"));
|
|
3492
|
+
} catch (error) {
|
|
3493
|
+
if (response === null) completedCalls += 1;
|
|
3494
|
+
observations.push(...(error.budgetObservations ?? []));
|
|
3495
|
+
throw error;
|
|
3496
|
+
}
|
|
2807
3497
|
}
|
|
2808
3498
|
const raw = payloads.join("\0");
|
|
2809
3499
|
clearBackoff(source.key);
|
|
3500
|
+
const batch = conditionalBatchResult(responses, raw, raw);
|
|
2810
3501
|
return {
|
|
2811
3502
|
raw,
|
|
2812
3503
|
completedCalls,
|
|
2813
3504
|
verdict: "ok",
|
|
3505
|
+
allNotModified: batch.allNotModified,
|
|
3506
|
+
stagedEntities: batch.stagedEntities,
|
|
3507
|
+
observations,
|
|
2814
3508
|
parse: () => {
|
|
2815
3509
|
const rows = mergeAlertRows(groups);
|
|
2816
3510
|
return {
|
|
@@ -2827,6 +3521,23 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2827
3521
|
// source's fixed note. Everything else -- an expired SAML session, a rate
|
|
2828
3522
|
// limit, a network drop -- surfaces as itself.
|
|
2829
3523
|
const verdict = classify(err);
|
|
3524
|
+
if (verdict === "unusable-output") {
|
|
3525
|
+
return {
|
|
3526
|
+
raw: "unusable-output",
|
|
3527
|
+
completedCalls,
|
|
3528
|
+
verdict,
|
|
3529
|
+
allNotModified: false,
|
|
3530
|
+
stagedEntities: new Map(),
|
|
3531
|
+
observations,
|
|
3532
|
+
parse: () => ({
|
|
3533
|
+
alerts: [],
|
|
3534
|
+
note: null,
|
|
3535
|
+
verdict,
|
|
3536
|
+
truncated: false,
|
|
3537
|
+
unusable: true,
|
|
3538
|
+
}),
|
|
3539
|
+
};
|
|
3540
|
+
}
|
|
2830
3541
|
const note = verdict === "unavailable" ? source.unavailable : `${source.name}: ${shortErr(err)}`;
|
|
2831
3542
|
const steps = pick(FAILURE_LADDER, verdict, null);
|
|
2832
3543
|
if (steps) {
|
|
@@ -2839,6 +3550,9 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
2839
3550
|
// whether it returned data, `[]`, or a 403.
|
|
2840
3551
|
completedCalls,
|
|
2841
3552
|
verdict,
|
|
3553
|
+
allNotModified: false,
|
|
3554
|
+
stagedEntities: new Map(),
|
|
3555
|
+
observations,
|
|
2842
3556
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
2843
3557
|
};
|
|
2844
3558
|
}
|
|
@@ -2856,7 +3570,11 @@ function severityRank(severity) {
|
|
|
2856
3570
|
return Object.hasOwn(SEVERITY_RANK, severity) ? SEVERITY_RANK[severity] : SEVERITY_RANK.unknown;
|
|
2857
3571
|
}
|
|
2858
3572
|
|
|
2859
|
-
async function fetchSecurity(signal
|
|
3573
|
+
async function fetchSecurity(signal, {
|
|
3574
|
+
entities = new Map(),
|
|
3575
|
+
force = false,
|
|
3576
|
+
previousRaw = null,
|
|
3577
|
+
} = {}) {
|
|
2860
3578
|
// Monotonic, not wall-clock. These deadlines measure *elapsed* time, and
|
|
2861
3579
|
// Date.now() can jump: a laptop resume or an NTP correction stepping the clock
|
|
2862
3580
|
// backwards would hold an alert source in backoff for up to an hour of apparent
|
|
@@ -2865,12 +3583,15 @@ async function fetchSecurity(signal) {
|
|
|
2865
3583
|
// for the mirror-image reason: a sleep gap is the thing it reports, and
|
|
2866
3584
|
// performance.now() does not advance across suspend.
|
|
2867
3585
|
const now = performance.now();
|
|
2868
|
-
const parts = await Promise.all(ALERT_SOURCES.map((
|
|
3586
|
+
const parts = await Promise.all(ALERT_SOURCES.map((source) =>
|
|
3587
|
+
fetchAlertSource(source, signal, now, { entities, force })));
|
|
3588
|
+
const allNotModified = parts.length > 0 && parts.every((part) => part.allNotModified);
|
|
3589
|
+
const stagedEntities = new Map(parts.flatMap((part) => [...part.stagedEntities]));
|
|
2869
3590
|
return {
|
|
2870
3591
|
// Joined with a NUL so a change in any one of the three shows up as a
|
|
2871
3592
|
// change in the combined payload, without risk of two different splits
|
|
2872
3593
|
// colliding on the same string.
|
|
2873
|
-
raw: parts.map((p) => p.raw).join("\0"),
|
|
3594
|
+
raw: allNotModified ? previousRaw : parts.map((p) => p.raw).join("\0"),
|
|
2874
3595
|
limit: ALERT_PER_PAGE,
|
|
2875
3596
|
// Summed from what each source reported rather than assumed to be
|
|
2876
3597
|
// ALERT_SOURCES.length: a source held off by backoff spawned nothing and
|
|
@@ -2879,7 +3600,10 @@ async function fetchSecurity(signal) {
|
|
|
2879
3600
|
graphqlSpent: 0,
|
|
2880
3601
|
measuredSuccess: parts.every((part) => part.verdict === "ok"),
|
|
2881
3602
|
rateLimited: parts.some((part) => part.verdict === "rate-limited"),
|
|
3603
|
+
stagedEntities,
|
|
3604
|
+
observations: parts.flatMap((part) => part.observations ?? []),
|
|
2882
3605
|
parse: () => {
|
|
3606
|
+
if (parts.some((part) => part.verdict === "unusable-output")) return { unusable: true };
|
|
2883
3607
|
const parsed = parts.map((p) => p.parse());
|
|
2884
3608
|
const alerts = parsed.flatMap((p) => p.alerts);
|
|
2885
3609
|
alerts.sort((a, b) => {
|
|
@@ -2890,6 +3614,7 @@ async function fetchSecurity(signal) {
|
|
|
2890
3614
|
return {
|
|
2891
3615
|
alerts,
|
|
2892
3616
|
notes: parsed.map((p) => p.note).filter(Boolean),
|
|
3617
|
+
unusable: parsed.some((p) => p.unusable),
|
|
2893
3618
|
// "Blind" is not the same as "switched off", and the tab bar could not
|
|
2894
3619
|
// tell them apart: fetchAlertSource never rejects, so a repo whose three
|
|
2895
3620
|
// endpoints all 403'd on an expired SAML session rendered
|
|
@@ -3267,19 +3992,55 @@ function normalizeRateBudgets(resources) {
|
|
|
3267
3992
|
]));
|
|
3268
3993
|
}
|
|
3269
3994
|
|
|
3270
|
-
async function
|
|
3995
|
+
async function readCoreBudget(signal, host = effectiveRuntimeHost(), etag = null) {
|
|
3271
3996
|
try {
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3997
|
+
const response = await ghApi(["user", ...apiHostArgs(host)], {
|
|
3998
|
+
signal,
|
|
3999
|
+
operation: "budget-core-observer",
|
|
4000
|
+
etag,
|
|
4001
|
+
});
|
|
4002
|
+
if (response.rateLimit?.resource !== "core") return null;
|
|
4003
|
+
return {
|
|
4004
|
+
budget: response.rateLimit,
|
|
4005
|
+
etag: response.etag ?? etag,
|
|
4006
|
+
receivedAt: Date.now(),
|
|
4007
|
+
};
|
|
4008
|
+
} catch (error) {
|
|
4009
|
+
const status = error?.apiResponse?.status;
|
|
4010
|
+
const rateLimit = pickRateLimit(error?.apiResponse?.headers);
|
|
4011
|
+
if (![403, 429].includes(status) || rateLimit?.resource !== "core") return null;
|
|
4012
|
+
return {
|
|
4013
|
+
budget: rateLimit,
|
|
4014
|
+
etag: error.apiResponse.headers.etag ?? etag,
|
|
4015
|
+
receivedAt: Date.now(),
|
|
4016
|
+
blocked: true,
|
|
4017
|
+
};
|
|
4018
|
+
}
|
|
4019
|
+
}
|
|
4020
|
+
|
|
4021
|
+
async function readSharedBudgetSources(signal, host = effectiveRuntimeHost(), {
|
|
4022
|
+
resources = RATE_RESOURCES,
|
|
4023
|
+
coreEtag = null,
|
|
4024
|
+
renewClaim = null,
|
|
4025
|
+
readGraphql = readRateLimitResources,
|
|
4026
|
+
readCore = readCoreBudget,
|
|
4027
|
+
} = {}) {
|
|
4028
|
+
const result = {};
|
|
4029
|
+
if (resources.includes("graphql")) {
|
|
4030
|
+
const rateResources = await readGraphql(signal, host).catch(() => null);
|
|
4031
|
+
const graphql = normalizeRateBudget(rateResources?.graphql);
|
|
4032
|
+
if (!graphql) return null;
|
|
4033
|
+
result.graphql = { budget: graphql, receivedAt: Date.now(), cost: 0 };
|
|
3282
4034
|
}
|
|
4035
|
+
if (resources.includes("core")) {
|
|
4036
|
+
// Each gh command is independently bounded, but two sequential commands
|
|
4037
|
+
// can outlive one claim. Extend the same nonce after the free GraphQL read
|
|
4038
|
+
// and before the one-cost bootstrap observer can start.
|
|
4039
|
+
if (resources.includes("graphql") && typeof renewClaim === "function" && !await renewClaim()) return null;
|
|
4040
|
+
const core = await readCore(signal, host, coreEtag);
|
|
4041
|
+
if (core) result.core = core;
|
|
4042
|
+
}
|
|
4043
|
+
return Object.keys(result).length > 0 ? result : null;
|
|
3283
4044
|
}
|
|
3284
4045
|
|
|
3285
4046
|
// Conservative unpaced demand at this configuration's floor, derived from the
|
|
@@ -3298,13 +4059,14 @@ function projectedHourlyCost(activeKey) {
|
|
|
3298
4059
|
function doctorProbePlan() {
|
|
3299
4060
|
return [
|
|
3300
4061
|
["Repository access", repoContextArgs(), "doctor:repository"],
|
|
3301
|
-
["Actions
|
|
4062
|
+
["Actions runs", ghApiArgs(actionsRunsArgs(MIN_RUN_LIMIT)), "doctor:actions-runs"],
|
|
4063
|
+
["Actions workflows", ghApiArgs(actionsWorkflowsArgs()), "doctor:actions-workflows"],
|
|
3302
4064
|
["Issues (issue list)", issuesArgs(), "doctor:issues"],
|
|
3303
4065
|
["Pull requests (pr list)", prsArgs(), "doctor:prs"],
|
|
3304
4066
|
...ALERT_SOURCES.flatMap((source) =>
|
|
3305
4067
|
alertRequestArgs(source).map((args, index) => [
|
|
3306
4068
|
index === 0 ? source.name : `${source.name} (priority ${index})`,
|
|
3307
|
-
args,
|
|
4069
|
+
ghApiArgs(args),
|
|
3308
4070
|
"doctor:security-endpoint",
|
|
3309
4071
|
]),
|
|
3310
4072
|
),
|
|
@@ -3349,8 +4111,8 @@ async function runDoctor() {
|
|
|
3349
4111
|
|
|
3350
4112
|
const probes = doctorProbePlan();
|
|
3351
4113
|
|
|
3352
|
-
// Free checks run first.
|
|
3353
|
-
//
|
|
4114
|
+
// Free checks run first. The display rate_limit read is report data only;
|
|
4115
|
+
// admission re-reads its GraphQL source after owning the shared claim.
|
|
3354
4116
|
const [ghVersion, authStatus, remote, remoteUrls] = await Promise.all([
|
|
3355
4117
|
captureGh(["--version"], "version"),
|
|
3356
4118
|
captureGh(["auth", "status"], "auth-status"),
|
|
@@ -3378,13 +4140,7 @@ async function runDoctor() {
|
|
|
3378
4140
|
demand: { core: 6, graphql: 2 },
|
|
3379
4141
|
});
|
|
3380
4142
|
try {
|
|
3381
|
-
|
|
3382
|
-
if (registered.ok && normalized?.core && normalized?.graphql) {
|
|
3383
|
-
const claim = claimProbe(scope, leaseId, nowMs);
|
|
3384
|
-
if (claim.value?.status === "claimed") {
|
|
3385
|
-
publishProbe(scope, leaseId, claim.value.nonce, normalized, nowMs);
|
|
3386
|
-
}
|
|
3387
|
-
}
|
|
4143
|
+
if (registered.ok) await refreshSharedBudget(scope, leaseId, undefined);
|
|
3388
4144
|
const admissionAt = Date.now();
|
|
3389
4145
|
const admittedProbes = probes.map(([name, args, operation], index) => {
|
|
3390
4146
|
const admitted = registered.ok
|
|
@@ -3460,7 +4216,7 @@ async function runDoctor() {
|
|
|
3460
4216
|
return field(
|
|
3461
4217
|
resource,
|
|
3462
4218
|
detail
|
|
3463
|
-
? `${detail.remaining} remaining, ${detail.reserve} reserved, reset ${new Date(detail.resetMs).toISOString()}`
|
|
4219
|
+
? `${detail.remaining} remaining, ${detail.reserve} reserved, reset ${new Date(detail.resetMs).toISOString()} (${detail.source})`
|
|
3464
4220
|
: "unavailable",
|
|
3465
4221
|
);
|
|
3466
4222
|
}),
|
|
@@ -3673,6 +4429,14 @@ function helpLines(maxRows) {
|
|
|
3673
4429
|
const all = keyTableLines();
|
|
3674
4430
|
if (all.length <= rows) return all;
|
|
3675
4431
|
if (rows === 1) return [`… ${all.length} keys: gh-glance --help`];
|
|
4432
|
+
if (rows === 4) {
|
|
4433
|
+
return [
|
|
4434
|
+
all[9],
|
|
4435
|
+
all[6],
|
|
4436
|
+
"Ent/jk Open the selected/Move the cursor",
|
|
4437
|
+
`… ${all.length - 4} more: gh-glance --help`,
|
|
4438
|
+
];
|
|
4439
|
+
}
|
|
3676
4440
|
const priority = [9, 6, 5, 3, 0, 7, 8, 1, 2, 4]
|
|
3677
4441
|
.slice(0, rows - 1)
|
|
3678
4442
|
.map((index) => all[index]);
|
|
@@ -3721,8 +4485,8 @@ from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
|
|
|
3721
4485
|
|
|
3722
4486
|
On a healthy single pane, the active tab is considered every ${REFRESH_MS / 1000}s and each
|
|
3723
4487
|
background tab about every ${(REFRESH_MS * BACKGROUND_EVERY) / 1000}s. Quota-consuming calls still
|
|
3724
|
-
need a shared, resource-specific grant that preserves a hard reserve.
|
|
3725
|
-
and manual refresh do not bypass that safety check.
|
|
4488
|
+
need a shared, resource-specific grant that preserves a hard reserve. Scheduled
|
|
4489
|
+
checks and manual refresh do not bypass that safety check.
|
|
3726
4490
|
|
|
3727
4491
|
Row icons are GitHub Octicons and need a Nerd Font. Without one, set
|
|
3728
4492
|
GH_GLANCE_ICONS=unicode for Unicode status glyphs and text row substitutes, or
|
|
@@ -4520,7 +5284,7 @@ const MemoSecurityRow = React.memo(SecurityRow);
|
|
|
4520
5284
|
const TABS = [
|
|
4521
5285
|
{
|
|
4522
5286
|
key: "actions",
|
|
4523
|
-
fetch: ({ signal, runLimit }) => fetchActions(runLimit, signal),
|
|
5287
|
+
fetch: ({ signal, runLimit, ...options }) => fetchActions(runLimit, signal, options),
|
|
4524
5288
|
label: "Actions",
|
|
4525
5289
|
short: "Actions",
|
|
4526
5290
|
header: ACTIONS_HEADER,
|
|
@@ -4550,7 +5314,7 @@ const TABS = [
|
|
|
4550
5314
|
},
|
|
4551
5315
|
{
|
|
4552
5316
|
key: "security",
|
|
4553
|
-
fetch: ({ signal }) => fetchSecurity(signal),
|
|
5317
|
+
fetch: ({ signal, ...options }) => fetchSecurity(signal, options),
|
|
4554
5318
|
label: "Security",
|
|
4555
5319
|
short: "Security",
|
|
4556
5320
|
header: SECURITY_HEADER,
|
|
@@ -4772,7 +5536,6 @@ function isRecord(value) {
|
|
|
4772
5536
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4773
5537
|
}
|
|
4774
5538
|
|
|
4775
|
-
const persistenceWaitCell = new Int32Array(new SharedArrayBuffer(4));
|
|
4776
5539
|
const PERSISTENCE_LOCK_WAIT_MS = 250;
|
|
4777
5540
|
const PERSISTENCE_STALE_LOCK_MS = 5000;
|
|
4778
5541
|
|
|
@@ -5501,16 +6264,42 @@ function governorWakeTimes(state, nowMs, floorMs, leaseId = null) {
|
|
|
5501
6264
|
};
|
|
5502
6265
|
}
|
|
5503
6266
|
|
|
5504
|
-
function
|
|
6267
|
+
function governorProtocolReady(refreshResult, snapshot, nowMs) {
|
|
5505
6268
|
if (!refreshResult?.ok || !snapshot?.ok) return false;
|
|
5506
6269
|
if (["waiting", "paused", "probe"].includes(refreshResult.value?.status)) return false;
|
|
5507
6270
|
if (snapshot.value.probeClaim || snapshot.value.probeOutcome?.status !== "healthy") return false;
|
|
6271
|
+
return RATE_RESOURCES.every((resource) => {
|
|
6272
|
+
const budget = snapshot.value.budgets[resource];
|
|
6273
|
+
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource);
|
|
6274
|
+
});
|
|
6275
|
+
}
|
|
6276
|
+
|
|
6277
|
+
function governorControlReady(refreshResult, snapshot, nowMs) {
|
|
6278
|
+
if (governorProtocolReady(refreshResult, snapshot, nowMs)) return true;
|
|
6279
|
+
// A losing probe claimant can exhaust its bounded reinspection just before
|
|
6280
|
+
// the winner publishes. Its return value is still `waiting`, but the locked
|
|
6281
|
+
// snapshot taken immediately afterwards is authoritative. Adopt only that
|
|
6282
|
+
// successful handoff; failures and every incomplete snapshot remain closed.
|
|
6283
|
+
return refreshResult?.ok && refreshResult.value?.status === "waiting" &&
|
|
6284
|
+
governorProtocolReady({ ok: true, value: { status: "published" } }, snapshot, nowMs);
|
|
6285
|
+
}
|
|
6286
|
+
|
|
6287
|
+
function tabEpochChanged(previous, next, key) {
|
|
6288
|
+
const costs = tabRequestCost(key);
|
|
6289
|
+
if (!costs || !isRecord(previous) || !isRecord(next)) return false;
|
|
6290
|
+
return RATE_RESOURCES.some((resource) =>
|
|
6291
|
+
costs[resource] > 0 && previous[resource] !== next[resource]);
|
|
6292
|
+
}
|
|
6293
|
+
|
|
6294
|
+
function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
|
|
6295
|
+
if (!governorProtocolReady(refreshResult, snapshot, nowMs)) return false;
|
|
5508
6296
|
const costs = tabRequestCost(activeKey);
|
|
5509
6297
|
return RATE_RESOURCES.every((resource) => {
|
|
5510
6298
|
if (costs[resource] <= 0) return true;
|
|
5511
6299
|
const budget = snapshot.value.budgets[resource];
|
|
5512
|
-
return budget && nowMs - budget.observedAt <=
|
|
5513
|
-
budget.blockUntil <= nowMs &&
|
|
6300
|
+
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
|
|
6301
|
+
budget.blockUntil <= nowMs &&
|
|
6302
|
+
availableForGrant({ budget, resource, nowMs }).mode === "open";
|
|
5514
6303
|
});
|
|
5515
6304
|
}
|
|
5516
6305
|
|
|
@@ -5582,8 +6371,8 @@ function pendingFailureIsTerminal(reason) {
|
|
|
5582
6371
|
return reason === "stale";
|
|
5583
6372
|
}
|
|
5584
6373
|
|
|
5585
|
-
function runtimeIntentGate(liveScheduling, { force = false } = {}) {
|
|
5586
|
-
return liveScheduling
|
|
6374
|
+
function runtimeIntentGate(liveScheduling, { force = false, protocolReady = false } = {}) {
|
|
6375
|
+
return liveScheduling || protocolReady
|
|
5587
6376
|
? { registerIntent: true, requestProbe: false }
|
|
5588
6377
|
: { registerIntent: false, requestProbe: force === true };
|
|
5589
6378
|
}
|
|
@@ -5635,6 +6424,13 @@ function rateLimitBlockDecision(results, resetMs) {
|
|
|
5635
6424
|
};
|
|
5636
6425
|
}
|
|
5637
6426
|
|
|
6427
|
+
function coordinationNotice(reason) {
|
|
6428
|
+
if (reason === "unknown-scope") return "Confirming your GitHub login…";
|
|
6429
|
+
if (reason === "block-unpublished") return "Holding until the rate-limit block is shared";
|
|
6430
|
+
if (reason === "busy" || reason === "stale") return "Coordinating with your other panes";
|
|
6431
|
+
return "Can't coordinate API use — retrying";
|
|
6432
|
+
}
|
|
6433
|
+
|
|
5638
6434
|
function retryRateLimitBlockPublication(pending, nowMs, publish) {
|
|
5639
6435
|
const blocks = Array.isArray(pending?.blocks) ? pending.blocks : [];
|
|
5640
6436
|
if (blocks.length === 0 || blocks.some((block) =>
|
|
@@ -5728,7 +6524,14 @@ async function runAdmittedOperation({ scope, leaseId, operation, priority = "man
|
|
|
5728
6524
|
|
|
5729
6525
|
function pollResultTransition({ key, previousRaw, raw, parse, limit, completedAt }) {
|
|
5730
6526
|
if (previousRaw === raw) return { kind: "unchanged", completedAt };
|
|
5731
|
-
|
|
6527
|
+
let value;
|
|
6528
|
+
try {
|
|
6529
|
+
value = parse();
|
|
6530
|
+
} catch (error) {
|
|
6531
|
+
if (isUnusableOutput(error)) return { kind: "unusable", nextRaw: null };
|
|
6532
|
+
throw error;
|
|
6533
|
+
}
|
|
6534
|
+
if (value?.unusable) return { kind: "unusable", nextRaw: null };
|
|
5732
6535
|
if (key === "security" && value?.blind) {
|
|
5733
6536
|
return {
|
|
5734
6537
|
kind: "blind",
|
|
@@ -5890,13 +6693,17 @@ const REMOTE_SETUP_NONINTERACTIVE_LINES = [
|
|
|
5890
6693
|
// Reserved so the hints never shift when the active tab changes state. Every
|
|
5891
6694
|
// status label below fits this fixed cell in both icon profiles.
|
|
5892
6695
|
const REFRESH_STATUS_WIDTH = 12;
|
|
6696
|
+
const WIDE_STATE_WIDTH = REFRESH_STATUS_WIDTH + 1 + 12;
|
|
6697
|
+
// StatusBar receives the drawable frame width, which is one column narrower
|
|
6698
|
+
// than the terminal. A 44-cell threshold preserves the 45-column contract.
|
|
6699
|
+
const WIDE_STATE_MIN_COLS = 44;
|
|
6700
|
+
const NOTICE_ROWS = 1;
|
|
5893
6701
|
|
|
5894
6702
|
const REFRESH_STATUS_GLYPHS = Object.freeze({
|
|
5895
6703
|
unicode: Object.freeze({
|
|
5896
6704
|
setup: "·",
|
|
5897
6705
|
checking: SPINNER[0],
|
|
5898
6706
|
paused: "‖",
|
|
5899
|
-
waiting: "·",
|
|
5900
6707
|
failed: "!",
|
|
5901
6708
|
limited: "?",
|
|
5902
6709
|
watching: "·",
|
|
@@ -5905,7 +6712,6 @@ const REFRESH_STATUS_GLYPHS = Object.freeze({
|
|
|
5905
6712
|
setup: ".",
|
|
5906
6713
|
checking: ".",
|
|
5907
6714
|
paused: "|",
|
|
5908
|
-
waiting: ".",
|
|
5909
6715
|
failed: "!",
|
|
5910
6716
|
limited: "?",
|
|
5911
6717
|
watching: ".",
|
|
@@ -5954,30 +6760,38 @@ function refreshStatus({
|
|
|
5954
6760
|
return status("paused", "paused", "Paused", "attention", false, detailKind);
|
|
5955
6761
|
}
|
|
5956
6762
|
if (["waiting", "pending", "probe"].includes(mode)) {
|
|
5957
|
-
const
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
6763
|
+
const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
|
|
6764
|
+
const watchingDetail = sharing
|
|
6765
|
+
? "sharing"
|
|
6766
|
+
: Number.isFinite(governorDecision?.notBefore) ? "next" : detailKind;
|
|
6767
|
+
return status("watching", "watching", "Watching", "inert", false, watchingDetail);
|
|
5961
6768
|
}
|
|
5962
6769
|
if (activeError) return status("failed", "failed", "Failed", "attention");
|
|
5963
6770
|
if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
|
|
5964
6771
|
return status("watching", "watching", "Watching", "inert");
|
|
5965
6772
|
}
|
|
5966
6773
|
|
|
5967
|
-
function
|
|
5968
|
-
if (!Number.isFinite(at)) return null;
|
|
5969
|
-
const
|
|
5970
|
-
if (
|
|
5971
|
-
|
|
6774
|
+
function statusInterval(at, nowMs = Date.now()) {
|
|
6775
|
+
if (!Number.isFinite(at) || !Number.isFinite(nowMs)) return null;
|
|
6776
|
+
const remaining = at - nowMs;
|
|
6777
|
+
if (remaining < 60_000) return "<1m";
|
|
6778
|
+
const minutes = Math.ceil(remaining / 60_000);
|
|
6779
|
+
return minutes > 99 ? "99m+" : `${minutes}m`;
|
|
5972
6780
|
}
|
|
5973
6781
|
|
|
5974
|
-
function statusDetailVariants(status, detail) {
|
|
6782
|
+
function statusDetailVariants(status, detail, nowMs) {
|
|
5975
6783
|
if (!status?.detailKind) return [];
|
|
5976
6784
|
if (status.detailKind === "probing") return ["probing"];
|
|
6785
|
+
if (status.detailKind === "sharing") {
|
|
6786
|
+
const sharing = sharedLaneProvenance(detail);
|
|
6787
|
+
return sharing.waitCause
|
|
6788
|
+
? [`sharing ${sharing.sharingCount}`]
|
|
6789
|
+
: [];
|
|
6790
|
+
}
|
|
5977
6791
|
const at = status.detailKind === "next" ? detail?.notBefore : detail?.resetMs;
|
|
5978
|
-
const
|
|
5979
|
-
if (!
|
|
5980
|
-
return [`${status.detailKind} ${
|
|
6792
|
+
const interval = statusInterval(at, nowMs);
|
|
6793
|
+
if (!interval) return [];
|
|
6794
|
+
return [`${status.detailKind} ${interval}`, interval];
|
|
5981
6795
|
}
|
|
5982
6796
|
|
|
5983
6797
|
function statusBarLayout({
|
|
@@ -5987,20 +6801,25 @@ function statusBarLayout({
|
|
|
5987
6801
|
status,
|
|
5988
6802
|
detail = null,
|
|
5989
6803
|
stale = null,
|
|
6804
|
+
nowMs = Date.now(),
|
|
5990
6805
|
version: currentVersion = "",
|
|
5991
6806
|
} = {}) {
|
|
5992
6807
|
const width = Number.isSafeInteger(cols) ? Math.max(0, cols) : 0;
|
|
5993
6808
|
const hints = Array.isArray(availableHints) ? availableHints : [];
|
|
5994
6809
|
const mandatory = hints.filter(isMandatoryHint);
|
|
5995
6810
|
const optional = hints.filter((hint) => !isMandatoryHint(hint));
|
|
5996
|
-
|
|
6811
|
+
const stateWidth = width >= WIDE_STATE_MIN_COLS
|
|
6812
|
+
? Math.min(width, WIDE_STATE_WIDTH)
|
|
6813
|
+
: Math.min(width, REFRESH_STATUS_WIDTH);
|
|
6814
|
+
let used = stateWidth;
|
|
5997
6815
|
const selectedHints = [];
|
|
5998
6816
|
const separatorWidth = () => selectedHints.length > 0 ? 1 : 0;
|
|
5999
6817
|
const addHint = (hint, showLabel) => {
|
|
6000
6818
|
const text = showLabel ? `${hint.label}: ${hint.keys}` : hint.keys;
|
|
6001
|
-
const
|
|
6819
|
+
const separator = separatorWidth();
|
|
6820
|
+
const cost = separator + [...text].length;
|
|
6002
6821
|
if (used + cost > width) return false;
|
|
6003
|
-
selectedHints.push({ ...hint, showLabel, text });
|
|
6822
|
+
selectedHints.push({ ...hint, showLabel, text, start: used + separator });
|
|
6004
6823
|
used += cost;
|
|
6005
6824
|
return true;
|
|
6006
6825
|
};
|
|
@@ -6020,20 +6839,17 @@ function statusBarLayout({
|
|
|
6020
6839
|
else addHint(hint, false);
|
|
6021
6840
|
}
|
|
6022
6841
|
|
|
6842
|
+
const payloadWidth = stateWidth > REFRESH_STATUS_WIDTH
|
|
6843
|
+
? stateWidth - REFRESH_STATUS_WIDTH - 1
|
|
6844
|
+
: 0;
|
|
6845
|
+
const staleText = typeof stale === "string" ? stale : null;
|
|
6846
|
+
const selectedStale = staleText && [...staleText].length <= payloadWidth ? staleText : null;
|
|
6023
6847
|
let selectedDetail = null;
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
selectedDetail = candidate;
|
|
6028
|
-
used += cost;
|
|
6029
|
-
break;
|
|
6030
|
-
}
|
|
6848
|
+
if (!selectedStale) {
|
|
6849
|
+
selectedDetail = statusDetailVariants(status, detail, nowMs)
|
|
6850
|
+
.find((candidate) => [...candidate].length <= payloadWidth) ?? null;
|
|
6031
6851
|
}
|
|
6032
6852
|
|
|
6033
|
-
const staleText = typeof stale === "string" ? stale : null;
|
|
6034
|
-
const selectedStale = staleText && used + 1 + staleText.length <= width ? staleText : null;
|
|
6035
|
-
if (selectedStale) used += 1 + selectedStale.length;
|
|
6036
|
-
|
|
6037
6853
|
for (const hint of optional) {
|
|
6038
6854
|
if (!addHint(hint, true)) addHint(hint, false);
|
|
6039
6855
|
}
|
|
@@ -6041,7 +6857,7 @@ function statusBarLayout({
|
|
|
6041
6857
|
const versionText = String(currentVersion ?? "");
|
|
6042
6858
|
const showVersion = versionText.length > 0 && used + 1 + versionText.length <= width;
|
|
6043
6859
|
return {
|
|
6044
|
-
|
|
6860
|
+
stateWidth,
|
|
6045
6861
|
hints: selectedHints,
|
|
6046
6862
|
mandatoryHints: selectedHints.filter(isMandatoryHint),
|
|
6047
6863
|
optionalHints: selectedHints.filter((hint) => !isMandatoryHint(hint)),
|
|
@@ -6055,11 +6871,13 @@ function statusBarLayout({
|
|
|
6055
6871
|
function freshnessDeadline({
|
|
6056
6872
|
lastOk,
|
|
6057
6873
|
refreshMs,
|
|
6874
|
+
grantedMs = null,
|
|
6058
6875
|
governorDecision = null,
|
|
6059
6876
|
currentEpochs = null,
|
|
6060
6877
|
} = {}) {
|
|
6061
6878
|
if (!Number.isFinite(lastOk) || !Number.isFinite(refreshMs) || refreshMs < 0) return null;
|
|
6062
|
-
const
|
|
6879
|
+
const cadenceMs = Number.isFinite(grantedMs) && grantedMs >= 0 ? grantedMs : refreshMs;
|
|
6880
|
+
const baseDeadline = lastOk + Math.max(STALE_AFTER_MS, cadenceMs * 6);
|
|
6063
6881
|
const decisionEpochs = governorDecision?.epochs;
|
|
6064
6882
|
const chargedEpochs = isRecord(decisionEpochs)
|
|
6065
6883
|
? Object.entries(decisionEpochs).filter(([, epoch]) => epoch !== null)
|
|
@@ -6075,6 +6893,16 @@ function freshnessDeadline({
|
|
|
6075
6893
|
: baseDeadline;
|
|
6076
6894
|
}
|
|
6077
6895
|
|
|
6896
|
+
function nextAdmittedCadence(previous, startedAt) {
|
|
6897
|
+
if (!Number.isFinite(startedAt)) return previous ?? { startedAt: null, grantedMs: null };
|
|
6898
|
+
return {
|
|
6899
|
+
startedAt,
|
|
6900
|
+
grantedMs: Number.isFinite(previous?.startedAt)
|
|
6901
|
+
? Math.max(0, startedAt - previous.startedAt)
|
|
6902
|
+
: null,
|
|
6903
|
+
};
|
|
6904
|
+
}
|
|
6905
|
+
|
|
6078
6906
|
function probingGovernorDecisions() {
|
|
6079
6907
|
return Object.fromEntries(TABS.map((candidate) => [candidate.key, {
|
|
6080
6908
|
mode: "waiting",
|
|
@@ -6099,6 +6927,8 @@ function sameVisibleGovernorDecision(left, right) {
|
|
|
6099
6927
|
"resource",
|
|
6100
6928
|
"probing",
|
|
6101
6929
|
"detailKind",
|
|
6930
|
+
"waitCause",
|
|
6931
|
+
"sharingCount",
|
|
6102
6932
|
"coordinationError",
|
|
6103
6933
|
"reservationId",
|
|
6104
6934
|
]) {
|
|
@@ -6149,6 +6979,7 @@ function StatusBar({
|
|
|
6149
6979
|
detail,
|
|
6150
6980
|
spin,
|
|
6151
6981
|
stale,
|
|
6982
|
+
nowMs,
|
|
6152
6983
|
interactive,
|
|
6153
6984
|
cols,
|
|
6154
6985
|
remoteSetup = false,
|
|
@@ -6190,6 +7021,7 @@ function StatusBar({
|
|
|
6190
7021
|
status: semanticStatus,
|
|
6191
7022
|
detail,
|
|
6192
7023
|
stale,
|
|
7024
|
+
nowMs,
|
|
6193
7025
|
version,
|
|
6194
7026
|
});
|
|
6195
7027
|
const profile = ICON_PROFILE === "ascii" ? "ascii" : "unicode";
|
|
@@ -6218,20 +7050,20 @@ function StatusBar({
|
|
|
6218
7050
|
{ flexDirection: "row" },
|
|
6219
7051
|
e(
|
|
6220
7052
|
Box,
|
|
6221
|
-
{ width: layout.
|
|
7053
|
+
{ width: layout.stateWidth, flexShrink: 0, flexDirection: "row" },
|
|
6222
7054
|
e(
|
|
6223
7055
|
Text,
|
|
6224
7056
|
{ color: statusColor, dimColor: semanticStatus.tone === "inert", wrap: "truncate-end" },
|
|
6225
7057
|
`${glyph} ${semanticStatus.label}`,
|
|
6226
7058
|
),
|
|
7059
|
+
layout.detail
|
|
7060
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { dimColor: true }, layout.detail))
|
|
7061
|
+
: null,
|
|
7062
|
+
layout.stale
|
|
7063
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, layout.stale))
|
|
7064
|
+
: null,
|
|
6227
7065
|
),
|
|
6228
7066
|
...layout.mandatoryHints.flatMap((hint, index) => renderHint(hint, index, "mandatory")),
|
|
6229
|
-
layout.detail
|
|
6230
|
-
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { dimColor: true }, layout.detail))
|
|
6231
|
-
: null,
|
|
6232
|
-
layout.stale
|
|
6233
|
-
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, layout.stale))
|
|
6234
|
-
: null,
|
|
6235
7067
|
...layout.optionalHints.flatMap((hint, index) => renderHint(hint, index, "optional")),
|
|
6236
7068
|
layout.version && e(Box, { flexGrow: 1 }),
|
|
6237
7069
|
layout.version && e(Text, { key: "version", dimColor: true, wrap: "truncate-end" }, layout.version),
|
|
@@ -6399,6 +7231,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6399
7231
|
const [requestStatuses, setRequestStatuses] = useState(() =>
|
|
6400
7232
|
Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
|
|
6401
7233
|
);
|
|
7234
|
+
const [visibleCoordinationCondition, setVisibleCoordinationCondition] = useState(null);
|
|
6402
7235
|
const [now, setNow] = useState(new Date());
|
|
6403
7236
|
const { rows, cols, useShortLabels } = useTerminalSize(stdout);
|
|
6404
7237
|
const [frame, setFrame] = useState(0);
|
|
@@ -6443,8 +7276,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6443
7276
|
// Counts the lines actually rendered, not the notes collected -- getting this
|
|
6444
7277
|
// wrong by one row is what makes ink repaint the whole frame.
|
|
6445
7278
|
const extraLines =
|
|
6446
|
-
|
|
6447
|
-
(activeGovernorDecision?.coordinationError && !remoteSetup ? 1 : 0) +
|
|
7279
|
+
NOTICE_ROWS +
|
|
6448
7280
|
(tab.key === "security" && !remoteSetup ? securityLines.length : 0);
|
|
6449
7281
|
// Reserve lines for: the tab bar and the divider under it (2), the panel's
|
|
6450
7282
|
// top and bottom edges (2), the column header and its separator (2), and
|
|
@@ -6475,7 +7307,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6475
7307
|
ANIMATE && tab.key === "actions" && (data.actions ?? []).some((r) => r.status === "in_progress");
|
|
6476
7308
|
|
|
6477
7309
|
const inFlightRef = useRef({});
|
|
7310
|
+
const admittedCadenceRef = useRef({});
|
|
6478
7311
|
const rawRef = useRef({});
|
|
7312
|
+
const entityRef = useRef(new Map());
|
|
6479
7313
|
// Last *successful* poll per tab, wall-clock. Wall-clock on purpose: a laptop
|
|
6480
7314
|
// sleeping is exactly the gap this is meant to report, and a monotonic clock
|
|
6481
7315
|
// does not advance across suspend. Never written on the failure path, or a
|
|
@@ -6992,6 +7826,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
6992
7826
|
status: "scheduled",
|
|
6993
7827
|
reservationId: `reservation:${pendingIntent.intentId}`,
|
|
6994
7828
|
...reservation,
|
|
7829
|
+
...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
|
|
6995
7830
|
}));
|
|
6996
7831
|
return;
|
|
6997
7832
|
}
|
|
@@ -7027,6 +7862,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7027
7862
|
force = false,
|
|
7028
7863
|
scope,
|
|
7029
7864
|
reservationId,
|
|
7865
|
+
admittedAt,
|
|
7030
7866
|
onSettled,
|
|
7031
7867
|
automaticStatusVisible = false,
|
|
7032
7868
|
} = {}) {
|
|
@@ -7035,6 +7871,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7035
7871
|
// background fetch -- and so a slow repo can't stack refreshes.
|
|
7036
7872
|
if (inFlightRef.current[key]) return Promise.resolve();
|
|
7037
7873
|
inFlightRef.current[key] = true;
|
|
7874
|
+
if (force) manualInFlight.add(key);
|
|
7875
|
+
admittedCadenceRef.current[key] = nextAdmittedCadence(
|
|
7876
|
+
admittedCadenceRef.current[key],
|
|
7877
|
+
admittedAt,
|
|
7878
|
+
);
|
|
7038
7879
|
clearForcedBackoffAfterStart(key, force, "started");
|
|
7039
7880
|
const visibleLoading = shouldShowFetchLoading({
|
|
7040
7881
|
hasData: dataRef.current[key] !== null,
|
|
@@ -7056,14 +7897,15 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7056
7897
|
}
|
|
7057
7898
|
return run()
|
|
7058
7899
|
.then((result) => {
|
|
7059
|
-
|
|
7060
|
-
? { outcome: "rejected" }
|
|
7900
|
+
settleReservationWithBudgetObservations(scope, leaseId, reservationId, result?.measuredSuccess === false
|
|
7901
|
+
? { outcome: "rejected", observations: result?.observations ?? [] }
|
|
7061
7902
|
: {
|
|
7062
7903
|
outcome: "measured-success",
|
|
7063
|
-
|
|
7904
|
+
actualCosts: {
|
|
7064
7905
|
core: result?.restSpent ?? REST_PER_FETCH[key] ?? 0,
|
|
7065
7906
|
graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
|
|
7066
7907
|
},
|
|
7908
|
+
observations: result?.observations ?? [],
|
|
7067
7909
|
}, Date.now());
|
|
7068
7910
|
if (result?.rateLimited) {
|
|
7069
7911
|
const blockAt = Date.now();
|
|
@@ -7075,7 +7917,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7075
7917
|
}], blockAt);
|
|
7076
7918
|
}
|
|
7077
7919
|
if (cancelled) return;
|
|
7078
|
-
const { raw, parse, limit } = result;
|
|
7920
|
+
const { raw, parse, limit, stagedEntities } = result;
|
|
7079
7921
|
// Identical payload: skip the parse *and* the state update. Returning
|
|
7080
7922
|
// the same state object makes React bail out of the re-render, so an
|
|
7081
7923
|
// idle repo stops redrawing the pane entirely.
|
|
@@ -7114,6 +7956,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7114
7956
|
force,
|
|
7115
7957
|
});
|
|
7116
7958
|
}
|
|
7959
|
+
publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
|
|
7117
7960
|
if (transition.kind === "unchanged") {
|
|
7118
7961
|
lastOkRef.current[key] = completedAt;
|
|
7119
7962
|
// Clear on the first success or a single failure latches the ladder.
|
|
@@ -7140,6 +7983,16 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7140
7983
|
}
|
|
7141
7984
|
return;
|
|
7142
7985
|
}
|
|
7986
|
+
// Empty or truncated JSON is not a user-actionable fetch error. Keep
|
|
7987
|
+
// the last-good rows and freshness clock, and clear the raw comparison
|
|
7988
|
+
// so the next admitted tick parses its response instead of taking the
|
|
7989
|
+
// identical-output fast path.
|
|
7990
|
+
if (transition.kind === "unusable") {
|
|
7991
|
+
if (key === "security") securityNextPollAt = 0;
|
|
7992
|
+
rawRef.current[key] = transition.nextRaw;
|
|
7993
|
+
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
7994
|
+
return;
|
|
7995
|
+
}
|
|
7143
7996
|
// Security fetches resolve each source independently so their notes
|
|
7144
7997
|
// remain visible. A blind result is still a failed observation: it
|
|
7145
7998
|
// must not replace known alerts with a false empty state or advance
|
|
@@ -7170,8 +8023,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7170
8023
|
});
|
|
7171
8024
|
})
|
|
7172
8025
|
.catch((err) => {
|
|
7173
|
-
|
|
8026
|
+
settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
|
|
7174
8027
|
outcome: governorOutcomeForError(err),
|
|
8028
|
+
observations: err?.budgetObservations ?? [],
|
|
7175
8029
|
}, Date.now());
|
|
7176
8030
|
if (cancelled || err?.name === "AbortError") return;
|
|
7177
8031
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
@@ -7207,6 +8061,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7207
8061
|
})
|
|
7208
8062
|
.finally(() => {
|
|
7209
8063
|
inFlightRef.current[key] = false;
|
|
8064
|
+
manualInFlight.delete(key);
|
|
7210
8065
|
onSettled?.();
|
|
7211
8066
|
if (!cancelled) {
|
|
7212
8067
|
setLoading((l) => (l[key] ? { ...l, [key]: false } : l));
|
|
@@ -7222,17 +8077,26 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7222
8077
|
: current);
|
|
7223
8078
|
}
|
|
7224
8079
|
}
|
|
8080
|
+
if (!cancelled && queuedManual.delete(key)) {
|
|
8081
|
+
void requestTab(key, "manual", { force: true });
|
|
8082
|
+
}
|
|
7225
8083
|
});
|
|
7226
8084
|
}
|
|
7227
8085
|
|
|
7228
8086
|
const leaseId = governorId();
|
|
7229
8087
|
const pending = new Map();
|
|
8088
|
+
// One manual handoff per tab. A keypress during an automatic request must
|
|
8089
|
+
// not be lost, but repeated keypresses during the forced request itself
|
|
8090
|
+
// must not create a trailing second batch.
|
|
8091
|
+
const queuedManual = new Set();
|
|
8092
|
+
const manualInFlight = new Set();
|
|
7230
8093
|
const wakeScheduler = createWakeScheduler();
|
|
7231
8094
|
let scope = null;
|
|
7232
8095
|
let cleanupScope = null;
|
|
7233
8096
|
let registeredScopeHash = null;
|
|
7234
8097
|
let remoteUrls = [];
|
|
7235
8098
|
let liveScheduling = false;
|
|
8099
|
+
let controlEpochs = null;
|
|
7236
8100
|
let activePollAt = Number.POSITIVE_INFINITY;
|
|
7237
8101
|
let backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7238
8102
|
let backgroundIndex = 0;
|
|
@@ -7244,6 +8108,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7244
8108
|
|
|
7245
8109
|
function failClosedRateLimit(key, reason) {
|
|
7246
8110
|
liveScheduling = false;
|
|
8111
|
+
controlEpochs = null;
|
|
7247
8112
|
activePollAt = Number.POSITIVE_INFINITY;
|
|
7248
8113
|
backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7249
8114
|
wakeScheduler.clear("data");
|
|
@@ -7400,6 +8265,12 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7400
8265
|
armFromState();
|
|
7401
8266
|
}
|
|
7402
8267
|
|
|
8268
|
+
function replaceActivePoll(kind, at) {
|
|
8269
|
+
if (["manual", "tab-switch"].includes(kind)) {
|
|
8270
|
+
activePollAt = at + runtime.refreshMs;
|
|
8271
|
+
}
|
|
8272
|
+
}
|
|
8273
|
+
|
|
7403
8274
|
function requestTab(key, kind = "active", { force = false } = {}) {
|
|
7404
8275
|
const signal = controller.signal;
|
|
7405
8276
|
const descriptor = tabForKey(key);
|
|
@@ -7408,7 +8279,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7408
8279
|
pauseCoordination(key, "block-unpublished");
|
|
7409
8280
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7410
8281
|
}
|
|
7411
|
-
if (inFlightRef.current[key])
|
|
8282
|
+
if (inFlightRef.current[key]) {
|
|
8283
|
+
if (force && !manualInFlight.has(key)) queuedManual.add(key);
|
|
8284
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
8285
|
+
}
|
|
7412
8286
|
if (!force && backoffActive(`tab:${key}`, monotonicNow)) {
|
|
7413
8287
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7414
8288
|
}
|
|
@@ -7421,20 +8295,39 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7421
8295
|
pauseCoordination(key, "unknown-scope");
|
|
7422
8296
|
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7423
8297
|
}
|
|
7424
|
-
|
|
8298
|
+
let protocolReady = liveScheduling;
|
|
8299
|
+
let resourceReady = liveScheduling;
|
|
8300
|
+
if (!liveScheduling || force) {
|
|
8301
|
+
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
8302
|
+
protocolReady = governorProtocolReady(
|
|
8303
|
+
{ ok: true, value: { status: "published" } },
|
|
8304
|
+
snapshot,
|
|
8305
|
+
nowMs,
|
|
8306
|
+
);
|
|
8307
|
+
resourceReady = governorDataReady(
|
|
8308
|
+
{ ok: true, value: { status: "published" } }, snapshot, key, nowMs);
|
|
8309
|
+
if (!liveScheduling && resourceReady) {
|
|
8310
|
+
liveScheduling = true;
|
|
8311
|
+
activePollAt = nowMs;
|
|
8312
|
+
backgroundPollAt = nowMs + 4 * runtime.refreshMs;
|
|
8313
|
+
armWake("heartbeat", nowMs + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
8314
|
+
}
|
|
8315
|
+
}
|
|
8316
|
+
const intentGate = runtimeIntentGate(liveScheduling, { force, protocolReady });
|
|
8317
|
+
if (intentGate.requestProbe || force && !resourceReady) {
|
|
8318
|
+
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
8319
|
+
const costs = tabRequestCost(key);
|
|
8320
|
+
for (const resource of RATE_RESOURCES) {
|
|
8321
|
+
const budget = snapshot.value?.budgets?.[resource];
|
|
8322
|
+
if (costs[resource] > 0 && budget) {
|
|
8323
|
+
requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
8324
|
+
}
|
|
8325
|
+
}
|
|
8326
|
+
armWake("control", nowMs + 1, controlWake);
|
|
8327
|
+
}
|
|
7425
8328
|
if (!intentGate.registerIntent) {
|
|
7426
8329
|
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
7427
8330
|
setTabWaiting(key, true);
|
|
7428
|
-
if (intentGate.requestProbe) {
|
|
7429
|
-
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
7430
|
-
const costs = tabRequestCost(key);
|
|
7431
|
-
for (const resource of RATE_RESOURCES) {
|
|
7432
|
-
const budget = snapshot.value?.budgets?.[resource];
|
|
7433
|
-
if (costs[resource] > 0 && budget) {
|
|
7434
|
-
requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
7435
|
-
}
|
|
7436
|
-
}
|
|
7437
|
-
}
|
|
7438
8331
|
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7439
8332
|
}
|
|
7440
8333
|
const existing = pending.get(key);
|
|
@@ -7460,8 +8353,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7460
8353
|
pauseCoordination(key, registered.reason);
|
|
7461
8354
|
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7462
8355
|
}
|
|
7463
|
-
pending.set(key, { intentId, kind, force, wasDeferred: false });
|
|
7464
8356
|
const decision = registered.value;
|
|
8357
|
+
pending.set(key, {
|
|
8358
|
+
intentId,
|
|
8359
|
+
kind,
|
|
8360
|
+
force,
|
|
8361
|
+
wasDeferred: false,
|
|
8362
|
+
...sharedLaneEvidence(decision),
|
|
8363
|
+
});
|
|
7465
8364
|
if (decision.status !== "scheduled" || decision.notBefore > nowMs) {
|
|
7466
8365
|
const item = pending.get(key);
|
|
7467
8366
|
if (item) item.wasDeferred = true;
|
|
@@ -7493,14 +8392,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7493
8392
|
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7494
8393
|
}
|
|
7495
8394
|
setTabWaiting(key, false);
|
|
8395
|
+
replaceActivePoll(kind, nowMs);
|
|
7496
8396
|
if (force) coordinator.invalidate();
|
|
7497
8397
|
return commit(
|
|
7498
8398
|
key,
|
|
7499
|
-
() => descriptor.fetch({
|
|
8399
|
+
() => descriptor.fetch({
|
|
8400
|
+
signal,
|
|
8401
|
+
runLimit: runLimitRef.current,
|
|
8402
|
+
entities: entityRef.current,
|
|
8403
|
+
force,
|
|
8404
|
+
previousRaw: rawRef.current[key] ?? null,
|
|
8405
|
+
}),
|
|
7500
8406
|
{
|
|
7501
8407
|
force,
|
|
7502
8408
|
scope: currentScope,
|
|
7503
8409
|
reservationId: decision.reservationId,
|
|
8410
|
+
admittedAt: nowMs,
|
|
7504
8411
|
automaticStatusVisible: false,
|
|
7505
8412
|
onSettled: () => finishPending(key, intentId),
|
|
7506
8413
|
},
|
|
@@ -7508,7 +8415,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7508
8415
|
}
|
|
7509
8416
|
fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
|
|
7510
8417
|
const requestedAt = Date.now();
|
|
7511
|
-
|
|
8418
|
+
// Manual work replaces the active poll that would otherwise be due. Move
|
|
8419
|
+
// that deadline before touching the governor so a data wake already in
|
|
8420
|
+
// this event-loop turn cannot start a second batch as soon as the forced
|
|
8421
|
+
// batch settles. The in-flight guard still coalesces work that overlaps.
|
|
8422
|
+
replaceActivePoll(kind, requestedAt);
|
|
7512
8423
|
const currentScope = ensureScope(requestedAt);
|
|
7513
8424
|
if (currentScope) {
|
|
7514
8425
|
heartbeatLease(currentScope, leaseId, tabRequestCost(key), requestedAt, key);
|
|
@@ -7538,7 +8449,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7538
8449
|
}
|
|
7539
8450
|
if (decision.value.status !== "scheduled" || decision.value.notBefore > nowMs) {
|
|
7540
8451
|
item.wasDeferred = true;
|
|
7541
|
-
setTabGovernorDecision(key, visibleGovernorDecision(
|
|
8452
|
+
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
8453
|
+
...decision.value,
|
|
8454
|
+
...sharedLaneProvenance(decision.value),
|
|
8455
|
+
}));
|
|
7542
8456
|
setTabWaiting(key, true);
|
|
7543
8457
|
armWake("data", decision.value.retryAt ?? decision.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7544
8458
|
continue;
|
|
@@ -7556,20 +8470,29 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7556
8470
|
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
7557
8471
|
...decision.value,
|
|
7558
8472
|
...started.value,
|
|
8473
|
+
...sharedLaneProvenance(decision.value),
|
|
7559
8474
|
}));
|
|
7560
8475
|
armWake("data", started.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7561
8476
|
continue;
|
|
7562
8477
|
}
|
|
7563
8478
|
setTabWaiting(key, false);
|
|
8479
|
+
replaceActivePoll(item.kind, nowMs);
|
|
7564
8480
|
if (item.force) coordinator.invalidate();
|
|
7565
8481
|
const descriptor = tabForKey(key);
|
|
7566
8482
|
void commit(
|
|
7567
8483
|
key,
|
|
7568
|
-
() => descriptor.fetch({
|
|
8484
|
+
() => descriptor.fetch({
|
|
8485
|
+
signal: controller.signal,
|
|
8486
|
+
runLimit: runLimitRef.current,
|
|
8487
|
+
entities: entityRef.current,
|
|
8488
|
+
force: item.force,
|
|
8489
|
+
previousRaw: rawRef.current[key] ?? null,
|
|
8490
|
+
}),
|
|
7569
8491
|
{
|
|
7570
8492
|
force: item.force,
|
|
7571
8493
|
scope: currentScope,
|
|
7572
8494
|
reservationId: decision.value.reservationId,
|
|
8495
|
+
admittedAt: nowMs,
|
|
7573
8496
|
automaticStatusVisible: item.wasDeferred && !item.force,
|
|
7574
8497
|
onSettled: () => finishPending(key, item.intentId),
|
|
7575
8498
|
},
|
|
@@ -7588,7 +8511,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7588
8511
|
const wakes = governorWakeTimes(snapshot.value, nowMs, runtime.refreshMs, leaseId);
|
|
7589
8512
|
const heldResources = Object.fromEntries(RATE_RESOURCES.map((resource) => {
|
|
7590
8513
|
const budget = snapshot.value.budgets[resource];
|
|
7591
|
-
const decision = budget
|
|
8514
|
+
const decision = budget
|
|
8515
|
+
? availableForGrant({ budget, resource, nowMs })
|
|
8516
|
+
: { mode: "paused" };
|
|
7592
8517
|
const held = !budget || budget.blockUntil > nowMs || decision.mode !== "open";
|
|
7593
8518
|
const retryAt = budget?.blockUntil > nowMs
|
|
7594
8519
|
? budget.blockUntil
|
|
@@ -7668,16 +8593,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7668
8593
|
attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
|
|
7669
8594
|
}
|
|
7670
8595
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7671
|
-
|
|
8596
|
+
const controlReady = pendingBlockPublications.size === 0 && governorControlReady(
|
|
7672
8597
|
refreshed,
|
|
7673
8598
|
snapshot,
|
|
7674
|
-
activeKey,
|
|
7675
8599
|
checkedAt,
|
|
7676
|
-
)
|
|
8600
|
+
);
|
|
8601
|
+
const activeEpochChanged = controlReady && controlEpochs !== null &&
|
|
8602
|
+
tabEpochChanged(controlEpochs, snapshot.value.epochs, activeKey);
|
|
8603
|
+
if (controlReady) controlEpochs = { ...snapshot.value.epochs };
|
|
8604
|
+
if (controlReady && (!liveScheduling || activeEpochChanged)) {
|
|
8605
|
+
const starting = !liveScheduling;
|
|
7677
8606
|
liveScheduling = true;
|
|
7678
8607
|
activePollAt = checkedAt + 1;
|
|
7679
|
-
|
|
7680
|
-
|
|
8608
|
+
if (starting) {
|
|
8609
|
+
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
8610
|
+
armWake("heartbeat", checkedAt + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
8611
|
+
}
|
|
7681
8612
|
}
|
|
7682
8613
|
if (!refreshed.ok) {
|
|
7683
8614
|
armWake("control", governorControlRetryAt(checkedAt, runtime.refreshMs), controlWake);
|
|
@@ -7714,12 +8645,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7714
8645
|
const snapshot = inspectGovernor(currentScope, checkedAt);
|
|
7715
8646
|
const activeKey = TABS[activeIndexRef.current].key;
|
|
7716
8647
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7717
|
-
|
|
8648
|
+
const controlReady = governorControlReady(
|
|
7718
8649
|
refreshed,
|
|
7719
8650
|
snapshot,
|
|
7720
|
-
activeKey,
|
|
7721
8651
|
checkedAt,
|
|
7722
|
-
)
|
|
8652
|
+
);
|
|
8653
|
+
if (controlReady) {
|
|
8654
|
+
controlEpochs = { ...snapshot.value.epochs };
|
|
7723
8655
|
liveScheduling = true;
|
|
7724
8656
|
activePollAt = checkedAt + 1;
|
|
7725
8657
|
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
@@ -7734,6 +8666,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7734
8666
|
void bootstrap();
|
|
7735
8667
|
return () => {
|
|
7736
8668
|
cancelled = true;
|
|
8669
|
+
queuedManual.clear();
|
|
8670
|
+
manualInFlight.clear();
|
|
7737
8671
|
wakeScheduler.clearAll();
|
|
7738
8672
|
for (const item of pending.values()) cancelIntent(cleanupScope, item.intentId, Date.now());
|
|
7739
8673
|
if (cleanupScope && registeredScopeHash) releaseLease(cleanupScope, leaseId);
|
|
@@ -7813,6 +8747,24 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7813
8747
|
|
|
7814
8748
|
const items = data[tab.key];
|
|
7815
8749
|
const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
|
|
8750
|
+
const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
|
|
8751
|
+
const coordinationReason = activeGovernorDecision?.reason ?? "unavailable";
|
|
8752
|
+
const coordinationCondition = coordinationError ? `${tab.key}\0${coordinationReason}` : null;
|
|
8753
|
+
useEffect(() => {
|
|
8754
|
+
setVisibleCoordinationCondition(null);
|
|
8755
|
+
if (coordinationCondition === null) return;
|
|
8756
|
+
const timer = setTimeout(
|
|
8757
|
+
() => setVisibleCoordinationCondition(coordinationCondition),
|
|
8758
|
+
COORDINATION_NOTICE_AFTER_MS,
|
|
8759
|
+
);
|
|
8760
|
+
return () => clearTimeout(timer);
|
|
8761
|
+
}, [coordinationCondition]);
|
|
8762
|
+
const showCoordinationNotice = coordinationCondition !== null &&
|
|
8763
|
+
visibleCoordinationCondition === coordinationCondition;
|
|
8764
|
+
const noticeLine = showCoordinationNotice
|
|
8765
|
+
? coordinationNotice(coordinationReason)
|
|
8766
|
+
: !remoteSetup && displayError ? displayError : "";
|
|
8767
|
+
const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
|
|
7816
8768
|
const spin = SPINNER[frame % SPINNER.length];
|
|
7817
8769
|
|
|
7818
8770
|
const counts = Object.fromEntries(
|
|
@@ -7867,6 +8819,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7867
8819
|
const staleAt = freshnessDeadline({
|
|
7868
8820
|
lastOk,
|
|
7869
8821
|
refreshMs: runtime.refreshMs,
|
|
8822
|
+
grantedMs: admittedCadenceRef.current[tab.key]?.grantedMs,
|
|
7870
8823
|
governorDecision: activeGovernorDecision,
|
|
7871
8824
|
currentEpochs: governorEpochs,
|
|
7872
8825
|
});
|
|
@@ -8075,20 +9028,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8075
9028
|
!showHelp &&
|
|
8076
9029
|
tooNarrow &&
|
|
8077
9030
|
e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
|
|
8078
|
-
|
|
8079
|
-
!tooNarrow &&
|
|
8080
|
-
!remoteSetup &&
|
|
8081
|
-
activeGovernorDecision?.coordinationError &&
|
|
8082
|
-
e(
|
|
8083
|
-
Text,
|
|
8084
|
-
{ color: ATTENTION, wrap: "truncate-end" },
|
|
8085
|
-
`API coordination unavailable (${activeGovernorDecision.reason}); retrying safely`,
|
|
8086
|
-
),
|
|
8087
|
-
!showHelp &&
|
|
8088
|
-
!tooNarrow &&
|
|
8089
|
-
!remoteSetup &&
|
|
8090
|
-
displayError &&
|
|
8091
|
-
e(Text, { color: ERROR_TEXT, wrap: "truncate-end" }, displayError),
|
|
9031
|
+
e(Text, { color: noticeTone, wrap: "truncate-end" }, noticeLine),
|
|
8092
9032
|
!showHelp &&
|
|
8093
9033
|
!tooNarrow &&
|
|
8094
9034
|
!remoteSetup &&
|
|
@@ -8167,7 +9107,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8167
9107
|
// Suppressed once icons are already ASCII, since then there is
|
|
8168
9108
|
// nothing to fix.
|
|
8169
9109
|
(USING_NERD_ICONS && iconHintDue ? " (icons blank? GH_GLANCE_ICONS=unicode)" : "")
|
|
8170
|
-
:
|
|
9110
|
+
: semanticStatus.kind === "paused" || waiting[tab.key]
|
|
8171
9111
|
? "waiting for API budget…"
|
|
8172
9112
|
: tab.key === "security" && securityNotes.length === ALERT_SOURCES.length
|
|
8173
9113
|
? "no alert sources available"
|
|
@@ -8184,6 +9124,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8184
9124
|
detail: activeGovernorDecision,
|
|
8185
9125
|
spin: showSpinner ? spin : null,
|
|
8186
9126
|
stale: staleLabel,
|
|
9127
|
+
nowMs: now.getTime(),
|
|
8187
9128
|
interactive,
|
|
8188
9129
|
cols: frameCols,
|
|
8189
9130
|
remoteSetup,
|
|
@@ -8461,6 +9402,9 @@ export {
|
|
|
8461
9402
|
BUDGET_SAFETY,
|
|
8462
9403
|
BUDGET_RESERVE_FRACTION,
|
|
8463
9404
|
BUDGET_SNAPSHOT_TTL_MS,
|
|
9405
|
+
GRAPHQL_BUDGET_SNAPSHOT_TTL_MS,
|
|
9406
|
+
budgetSnapshotTtl,
|
|
9407
|
+
currentSharedLaneProvenance,
|
|
8464
9408
|
GOVERNOR_HEARTBEAT_MS,
|
|
8465
9409
|
GOVERNOR_LEASE_TTL_MS,
|
|
8466
9410
|
GOVERNOR_PROBE_LEASE_MS,
|
|
@@ -8513,17 +9457,21 @@ export {
|
|
|
8513
9457
|
cancelIntent,
|
|
8514
9458
|
startReservation,
|
|
8515
9459
|
completeReservation,
|
|
9460
|
+
settleReservationWithBudgetObservations,
|
|
8516
9461
|
recordResourceBlock,
|
|
8517
9462
|
releaseLease,
|
|
8518
9463
|
inspectGovernor,
|
|
8519
9464
|
governorHealth,
|
|
8520
9465
|
refreshSharedBudget,
|
|
9466
|
+
readCoreBudget,
|
|
9467
|
+
readSharedBudgetSources,
|
|
8521
9468
|
safe,
|
|
8522
9469
|
shortErr,
|
|
8523
9470
|
isUnavailable,
|
|
8524
9471
|
isRateLimited,
|
|
8525
9472
|
isAuthProblem,
|
|
8526
9473
|
isMissingRemote,
|
|
9474
|
+
isUnusableOutput,
|
|
8527
9475
|
forwardSignalToChild,
|
|
8528
9476
|
toTabError,
|
|
8529
9477
|
formatTabError,
|
|
@@ -8540,6 +9488,19 @@ export {
|
|
|
8540
9488
|
BACKOFF_STEPS_MS,
|
|
8541
9489
|
redact,
|
|
8542
9490
|
classify,
|
|
9491
|
+
parseJsonOutput,
|
|
9492
|
+
parseGhApiResponse,
|
|
9493
|
+
pickRateLimit,
|
|
9494
|
+
ghApiArgs,
|
|
9495
|
+
ghApi,
|
|
9496
|
+
actionsRunsArgs,
|
|
9497
|
+
actionsWorkflowsArgs,
|
|
9498
|
+
parseActionsBodies,
|
|
9499
|
+
entityKey,
|
|
9500
|
+
fetchConditionalEntity,
|
|
9501
|
+
conditionalBatchResult,
|
|
9502
|
+
publishStagedEntities,
|
|
9503
|
+
fetchAlertSource,
|
|
8543
9504
|
formatAge,
|
|
8544
9505
|
formatDuration,
|
|
8545
9506
|
usableSize,
|
|
@@ -8580,6 +9541,8 @@ export {
|
|
|
8580
9541
|
pollSchedule,
|
|
8581
9542
|
retryPollAfterAdmissionFailure,
|
|
8582
9543
|
governorWakeTimes,
|
|
9544
|
+
governorControlReady,
|
|
9545
|
+
tabEpochChanged,
|
|
8583
9546
|
governorDataReady,
|
|
8584
9547
|
governorControlRetryAt,
|
|
8585
9548
|
createWakeScheduler,
|
|
@@ -8587,6 +9550,7 @@ export {
|
|
|
8587
9550
|
pendingFailureIsTerminal,
|
|
8588
9551
|
runtimeIntentGate,
|
|
8589
9552
|
rateLimitBlockDecision,
|
|
9553
|
+
coordinationNotice,
|
|
8590
9554
|
mergeRateLimitBlockPublications,
|
|
8591
9555
|
hydrateRateLimitBlockPublication,
|
|
8592
9556
|
retryRateLimitBlockPublication,
|
|
@@ -8622,8 +9586,10 @@ export {
|
|
|
8622
9586
|
summarizeDoctorEnv,
|
|
8623
9587
|
widthStatusText,
|
|
8624
9588
|
refreshStatus,
|
|
9589
|
+
statusInterval,
|
|
8625
9590
|
statusBarLayout,
|
|
8626
9591
|
freshnessDeadline,
|
|
9592
|
+
nextAdmittedCadence,
|
|
8627
9593
|
probingGovernorDecisions,
|
|
8628
9594
|
retainDeferredGovernorHold,
|
|
8629
9595
|
REFRESH_STATUS_GLYPHS,
|
|
@@ -8642,4 +9608,5 @@ export {
|
|
|
8642
9608
|
REMOTE_SETUP_NONINTERACTIVE_LINES,
|
|
8643
9609
|
VERDICT_REMEDY,
|
|
8644
9610
|
FAILURE_LADDER,
|
|
9611
|
+
COORDINATION_NOTICE_AFTER_MS,
|
|
8645
9612
|
};
|