ever-works-node 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +1421 -613
- package/package.json +1 -1
package/cli.js
CHANGED
|
@@ -3767,10 +3767,14 @@ var USER_SELECTABLE_WORK_KINDS = [
|
|
|
3767
3767
|
"repo",
|
|
3768
3768
|
"app"
|
|
3769
3769
|
];
|
|
3770
|
-
var WORK_KINDS = [...USER_SELECTABLE_WORK_KINDS, "company", "campaign", "default"];
|
|
3770
|
+
var WORK_KINDS = [...USER_SELECTABLE_WORK_KINDS, "company", "campaign", "project", "default"];
|
|
3771
3771
|
var WORK_KIND_SET = new Set(WORK_KINDS);
|
|
3772
3772
|
var GOAL_KINDS = ["metric", "delivery"];
|
|
3773
3773
|
var GOAL_KIND_SET = new Set(GOAL_KINDS);
|
|
3774
|
+
var RECORD_VISIBILITIES = ["private", "team", "organization"];
|
|
3775
|
+
var RECORD_VISIBILITY_SET = new Set(RECORD_VISIBILITIES);
|
|
3776
|
+
var RECORD_LIST_VIEWS = ["mine", "shared", "organization"];
|
|
3777
|
+
var RECORD_LIST_VIEW_SET = new Set(RECORD_LIST_VIEWS);
|
|
3774
3778
|
var APP_DEPENDENCY_BACKUP_OVERDUE_MS = 26 * 36e5;
|
|
3775
3779
|
var APP_DEPENDENCY_MANAGED = {
|
|
3776
3780
|
pgRoleConnectionLimit: 20,
|
|
@@ -5002,6 +5006,41 @@ function resolveExclusiveAgentCredentials(grantedNames, env) {
|
|
|
5002
5006
|
return { names: grantedNames.filter((name) => !dropped.has(name)), notes };
|
|
5003
5007
|
}
|
|
5004
5008
|
__name(resolveExclusiveAgentCredentials, "resolveExclusiveAgentCredentials");
|
|
5009
|
+
var CONVENTIONAL_SUBJECT = /^([a-z]+)(\([^)\n]{1,40}\))?(!)?:\s+(\S.*)$/;
|
|
5010
|
+
function classifyFleetChangedPaths(paths) {
|
|
5011
|
+
const list = (paths ?? []).map((path2) => path2.replace(/\\/g, "/").trim()).filter(Boolean);
|
|
5012
|
+
if (list.length === 0) return "none";
|
|
5013
|
+
if (list.every(isDocsPath)) return "docs";
|
|
5014
|
+
if (list.every(isTestPath)) return "test";
|
|
5015
|
+
return "mixed";
|
|
5016
|
+
}
|
|
5017
|
+
__name(classifyFleetChangedPaths, "classifyFleetChangedPaths");
|
|
5018
|
+
function retypeFleetCommitMessage(message, changedPaths) {
|
|
5019
|
+
if (typeof message !== "string" || !message) return message;
|
|
5020
|
+
const newline = message.indexOf("\n");
|
|
5021
|
+
const firstLine = newline === -1 ? message : message.slice(0, newline);
|
|
5022
|
+
const rest = newline === -1 ? "" : message.slice(newline);
|
|
5023
|
+
const parsed = CONVENTIONAL_SUBJECT.exec(firstLine);
|
|
5024
|
+
if (!parsed) return message;
|
|
5025
|
+
const change = classifyFleetChangedPaths(changedPaths);
|
|
5026
|
+
const type = change === "docs" ? "docs" : change === "test" ? "test" : null;
|
|
5027
|
+
if (!type || parsed[1] === type) return message;
|
|
5028
|
+
return `${type}${firstLine.slice(parsed[1].length)}${rest}`;
|
|
5029
|
+
}
|
|
5030
|
+
__name(retypeFleetCommitMessage, "retypeFleetCommitMessage");
|
|
5031
|
+
var DOC_EXTENSIONS = /\.(?:md|mdx|markdown|rst|adoc|txt)$/i;
|
|
5032
|
+
var DOC_DIRECTORY = /(?:^|\/)(?:docs?|documentation)\//i;
|
|
5033
|
+
var DOC_FILENAMES = /(?:^|\/)(?:readme|changelog|contributing|license|licence|notice|authors|code_of_conduct)(?:\.[a-z]+)?$/i;
|
|
5034
|
+
function isDocsPath(path2) {
|
|
5035
|
+
return DOC_EXTENSIONS.test(path2) || DOC_DIRECTORY.test(path2) || DOC_FILENAMES.test(path2);
|
|
5036
|
+
}
|
|
5037
|
+
__name(isDocsPath, "isDocsPath");
|
|
5038
|
+
var TEST_FILE = /(?:^|[./_-])(?:spec|test|tests|e2e)\.[cm]?[jt]sx?$|_test\.(?:go|py|rb)$|(?:^|\/)test_[^/]+\.py$/i;
|
|
5039
|
+
var TEST_DIRECTORY = /(?:^|\/)(?:__tests__|__snapshots__|__mocks__|__fixtures__|tests?|e2e|spec|fixtures)\//i;
|
|
5040
|
+
function isTestPath(path2) {
|
|
5041
|
+
return TEST_FILE.test(path2) || TEST_DIRECTORY.test(path2) || /\.snap$/i.test(path2);
|
|
5042
|
+
}
|
|
5043
|
+
__name(isTestPath, "isTestPath");
|
|
5005
5044
|
var INBOX_MAX_TITLE_CHARS = 300;
|
|
5006
5045
|
var FLEET_TASK_WORKSPACE_MAX_MOUNTS = 8;
|
|
5007
5046
|
var FLEET_TASK_WORKSPACE_MOUNT_DIR_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,62}[A-Za-z0-9_-])?$/;
|
|
@@ -5515,6 +5554,275 @@ function truncateToUtf8Bytes(value, maxBytes) {
|
|
|
5515
5554
|
return { value: decoded.replace(/\uFFFD+$/, "").trimEnd(), truncated: true };
|
|
5516
5555
|
}
|
|
5517
5556
|
__name(truncateToUtf8Bytes, "truncateToUtf8Bytes");
|
|
5557
|
+
var FLEET_JOB_FAILURE_KIND_MODEL_USAGE_LIMIT = "model_usage_limit";
|
|
5558
|
+
var FLEET_MODEL_USAGE_LIMIT_FALLBACK_MS = 60 * 60 * 1e3;
|
|
5559
|
+
var FLEET_MODEL_USAGE_LIMIT_MIN_MS = 60 * 1e3;
|
|
5560
|
+
var FLEET_MODEL_USAGE_LIMIT_MAX_MS = 8 * 24 * 60 * 60 * 1e3;
|
|
5561
|
+
var FLEET_MODEL_USAGE_LIMIT_RESET_SLACK_MS = 60 * 1e3;
|
|
5562
|
+
var FLEET_MODEL_USAGE_LIMIT_MAX_MESSAGE_LENGTH = 300;
|
|
5563
|
+
var LIMIT_SIGNATURES = [
|
|
5564
|
+
{ pattern: /\bhit (?:your|the) (?:[\w-]+ )?(?:usage |rate )?limit\b/i, strict: true },
|
|
5565
|
+
{
|
|
5566
|
+
pattern: /\b(?:usage|weekly|daily|session|5[- ]hour|five[- ]hour) limit (?:has been )?(?:reached|exceeded|hit)\b/i,
|
|
5567
|
+
strict: true
|
|
5568
|
+
},
|
|
5569
|
+
{ pattern: /\busage[_ ]limit[_ ](?:reached|exceeded)\b/i, strict: true },
|
|
5570
|
+
{ pattern: /\brate_limit_error\b/, strict: true },
|
|
5571
|
+
{ pattern: /\bexceeded retry limit,? last status:? 429\b/i, strict: true },
|
|
5572
|
+
{ pattern: /\b429\b[^\n]{0,40}\btoo many requests\b/i, strict: true },
|
|
5573
|
+
{ pattern: /\bAPI Error:? 429\b/i, strict: true },
|
|
5574
|
+
{ pattern: /\brate[- ]limit(?:ed)?\b[^\n]{0,20}\b(?:reached|exceeded)\b/i, strict: false },
|
|
5575
|
+
{ pattern: /\bout of (?:extra )?usage\b/i, strict: false }
|
|
5576
|
+
];
|
|
5577
|
+
function detectFleetModelUsageLimit(text, options = {}) {
|
|
5578
|
+
if (typeof text !== "string" || !text.trim()) return null;
|
|
5579
|
+
const strict = options.strict === true;
|
|
5580
|
+
const match = LIMIT_SIGNATURES.find((signature) => (!strict || signature.strict) && signature.pattern.test(text));
|
|
5581
|
+
if (!match) return null;
|
|
5582
|
+
const nowMs = toEpochMs(options.now);
|
|
5583
|
+
const provider = typeof options.provider === "string" && options.provider.trim() ? options.provider.trim() : "claude-code";
|
|
5584
|
+
const resetsAtMs = parseResetInstant(text, nowMs, options.timeZone);
|
|
5585
|
+
const resetsAt = resetsAtMs !== null && resetsAtMs > nowMs ? resetsAtMs : null;
|
|
5586
|
+
return {
|
|
5587
|
+
provider,
|
|
5588
|
+
window: classifyWindow(text),
|
|
5589
|
+
resetsAt: resetsAt === null ? null : new Date(resetsAt).toISOString(),
|
|
5590
|
+
retryAfter: new Date(
|
|
5591
|
+
clampRetryAfterMs(
|
|
5592
|
+
resetsAt === null ? nowMs + FLEET_MODEL_USAGE_LIMIT_FALLBACK_MS : resetsAt + FLEET_MODEL_USAGE_LIMIT_RESET_SLACK_MS,
|
|
5593
|
+
nowMs
|
|
5594
|
+
)
|
|
5595
|
+
).toISOString(),
|
|
5596
|
+
message: describeLimitSentence(text, match.pattern)
|
|
5597
|
+
};
|
|
5598
|
+
}
|
|
5599
|
+
__name(detectFleetModelUsageLimit, "detectFleetModelUsageLimit");
|
|
5600
|
+
function clampRetryAfterMs(retryAfterMs, nowMs) {
|
|
5601
|
+
const floor = nowMs + FLEET_MODEL_USAGE_LIMIT_MIN_MS;
|
|
5602
|
+
const ceiling = nowMs + FLEET_MODEL_USAGE_LIMIT_MAX_MS;
|
|
5603
|
+
if (!Number.isFinite(retryAfterMs)) return nowMs + FLEET_MODEL_USAGE_LIMIT_FALLBACK_MS;
|
|
5604
|
+
return Math.min(Math.max(retryAfterMs, floor), ceiling);
|
|
5605
|
+
}
|
|
5606
|
+
__name(clampRetryAfterMs, "clampRetryAfterMs");
|
|
5607
|
+
function describeFleetModelProvider(provider) {
|
|
5608
|
+
switch (provider) {
|
|
5609
|
+
case "claude-code":
|
|
5610
|
+
return "Claude Code";
|
|
5611
|
+
case "codex":
|
|
5612
|
+
return "Codex";
|
|
5613
|
+
default:
|
|
5614
|
+
return provider;
|
|
5615
|
+
}
|
|
5616
|
+
}
|
|
5617
|
+
__name(describeFleetModelProvider, "describeFleetModelProvider");
|
|
5618
|
+
function describeFleetModelUsageLimitWindow(window) {
|
|
5619
|
+
switch (window) {
|
|
5620
|
+
case "weekly":
|
|
5621
|
+
return "weekly limit";
|
|
5622
|
+
case "session":
|
|
5623
|
+
return "5-hour limit";
|
|
5624
|
+
case "daily":
|
|
5625
|
+
return "daily limit";
|
|
5626
|
+
case "rate":
|
|
5627
|
+
return "rate limit";
|
|
5628
|
+
default:
|
|
5629
|
+
return "usage limit";
|
|
5630
|
+
}
|
|
5631
|
+
}
|
|
5632
|
+
__name(describeFleetModelUsageLimitWindow, "describeFleetModelUsageLimitWindow");
|
|
5633
|
+
function toEpochMs(value) {
|
|
5634
|
+
if (value instanceof Date) return value.getTime();
|
|
5635
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
5636
|
+
return Date.now();
|
|
5637
|
+
}
|
|
5638
|
+
__name(toEpochMs, "toEpochMs");
|
|
5639
|
+
function classifyWindow(text) {
|
|
5640
|
+
if (/\bweekly\b|\bper week\b/i.test(text)) return "weekly";
|
|
5641
|
+
if (/\b(?:5|five)[- ]hour\b|\bsession limit\b/i.test(text)) return "session";
|
|
5642
|
+
if (/\bdaily\b|\bper day\b/i.test(text)) return "daily";
|
|
5643
|
+
if (/\b429\b|rate_limit_error|\brate[- ]limit/i.test(text)) return "rate";
|
|
5644
|
+
return "usage";
|
|
5645
|
+
}
|
|
5646
|
+
__name(classifyWindow, "classifyWindow");
|
|
5647
|
+
function trimSentence(text) {
|
|
5648
|
+
const clean = text.replace(/[\u0000-\u001f\u007f]+/g, " ").replace(/\s+/g, " ").trim();
|
|
5649
|
+
const points = Array.from(clean);
|
|
5650
|
+
return points.length > FLEET_MODEL_USAGE_LIMIT_MAX_MESSAGE_LENGTH ? `${points.slice(0, FLEET_MODEL_USAGE_LIMIT_MAX_MESSAGE_LENGTH - 1).join("").trimEnd()}\u2026` : clean;
|
|
5651
|
+
}
|
|
5652
|
+
__name(trimSentence, "trimSentence");
|
|
5653
|
+
function describeLimitSentence(text, pattern) {
|
|
5654
|
+
const line = text.split(/\r?\n/).find((candidate) => pattern.test(candidate));
|
|
5655
|
+
return trimSentence(line ?? text);
|
|
5656
|
+
}
|
|
5657
|
+
__name(describeLimitSentence, "describeLimitSentence");
|
|
5658
|
+
var MONTHS = {
|
|
5659
|
+
jan: 0,
|
|
5660
|
+
feb: 1,
|
|
5661
|
+
mar: 2,
|
|
5662
|
+
apr: 3,
|
|
5663
|
+
may: 4,
|
|
5664
|
+
jun: 5,
|
|
5665
|
+
jul: 6,
|
|
5666
|
+
aug: 7,
|
|
5667
|
+
sep: 8,
|
|
5668
|
+
sept: 8,
|
|
5669
|
+
oct: 9,
|
|
5670
|
+
nov: 10,
|
|
5671
|
+
dec: 11
|
|
5672
|
+
};
|
|
5673
|
+
function parseResetInstant(text, nowMs, timeZone) {
|
|
5674
|
+
const epoch = /usage limit reached\|(\d{10,13})\b/i.exec(text);
|
|
5675
|
+
if (epoch) {
|
|
5676
|
+
const value = Number(epoch[1]);
|
|
5677
|
+
return epoch[1].length >= 13 ? value : value * 1e3;
|
|
5678
|
+
}
|
|
5679
|
+
const iso = /\b(?:resets?|reset at|try again at|available again at|until)\s+(?:at\s+)?(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2}))/i.exec(
|
|
5680
|
+
text
|
|
5681
|
+
);
|
|
5682
|
+
if (iso) {
|
|
5683
|
+
const value = Date.parse(iso[1]);
|
|
5684
|
+
if (Number.isFinite(value)) return value;
|
|
5685
|
+
}
|
|
5686
|
+
const relative6 = /\b(?:resets?|reset|try again|available again|retry)\s+in\s+((?:\d+\s*(?:d|days?|h|hrs?|hours?|m|mins?|minutes?|s|secs?|seconds?)\b(?:[\s,]+(?:and )?)?)+)/i.exec(
|
|
5687
|
+
text
|
|
5688
|
+
);
|
|
5689
|
+
if (relative6) {
|
|
5690
|
+
const ms = parseDuration(relative6[1]);
|
|
5691
|
+
if (ms > 0) return nowMs + ms;
|
|
5692
|
+
}
|
|
5693
|
+
const wall = /\b(?:resets?|reset at|will reset at|try again at|available again at)\s+(?:at\s+|on\s+)?(?:(jan|feb|mar|apr|may|jun|jul|aug|sept?|oct|nov|dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(?:at\s+)?)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\b(?:\s*\(([A-Za-z_]+(?:\/[A-Za-z0-9_+-]+){0,2}|UTC|GMT)\))?/i.exec(
|
|
5694
|
+
text
|
|
5695
|
+
);
|
|
5696
|
+
if (wall) {
|
|
5697
|
+
const [, monthName, dayText, hourText, minuteText, meridiem, zoneText] = wall;
|
|
5698
|
+
let hour = Number(hourText);
|
|
5699
|
+
const minute = minuteText ? Number(minuteText) : 0;
|
|
5700
|
+
if (!meridiem && !minuteText && !monthName) {
|
|
5701
|
+
return null;
|
|
5702
|
+
}
|
|
5703
|
+
if (meridiem) {
|
|
5704
|
+
if (hour < 1 || hour > 12) return null;
|
|
5705
|
+
hour = hour % 12 + (meridiem.toLowerCase() === "pm" ? 12 : 0);
|
|
5706
|
+
}
|
|
5707
|
+
if (hour > 23 || minute > 59) return null;
|
|
5708
|
+
const zone = resolveZone(zoneText) ?? resolveZone(timeZone) ?? "UTC";
|
|
5709
|
+
const today = zonedParts(nowMs, zone);
|
|
5710
|
+
if (monthName) {
|
|
5711
|
+
const month = MONTHS[monthName.toLowerCase()];
|
|
5712
|
+
const day = Number(dayText);
|
|
5713
|
+
if (month === void 0 || day < 1 || day > 31) return null;
|
|
5714
|
+
let candidate2 = zonedWallTimeToUtc(today.year, month, day, hour, minute, zone);
|
|
5715
|
+
if (candidate2 < nowMs - 24 * 60 * 60 * 1e3) {
|
|
5716
|
+
candidate2 = zonedWallTimeToUtc(today.year + 1, month, day, hour, minute, zone);
|
|
5717
|
+
}
|
|
5718
|
+
return candidate2;
|
|
5719
|
+
}
|
|
5720
|
+
let candidate = zonedWallTimeToUtc(today.year, today.month, today.day, hour, minute, zone);
|
|
5721
|
+
if (candidate <= nowMs) {
|
|
5722
|
+
candidate = zonedWallTimeToUtc(today.year, today.month, today.day + 1, hour, minute, zone);
|
|
5723
|
+
}
|
|
5724
|
+
return candidate;
|
|
5725
|
+
}
|
|
5726
|
+
return null;
|
|
5727
|
+
}
|
|
5728
|
+
__name(parseResetInstant, "parseResetInstant");
|
|
5729
|
+
function parseDuration(text) {
|
|
5730
|
+
let total = 0;
|
|
5731
|
+
let index = 0;
|
|
5732
|
+
const length = Math.min(text.length, 200);
|
|
5733
|
+
while (index < length) {
|
|
5734
|
+
if (!isDigit(text.charCodeAt(index))) {
|
|
5735
|
+
index += 1;
|
|
5736
|
+
continue;
|
|
5737
|
+
}
|
|
5738
|
+
let value = 0;
|
|
5739
|
+
while (index < length && isDigit(text.charCodeAt(index))) {
|
|
5740
|
+
value = Math.min(value * 10 + (text.charCodeAt(index) - 48), 1e6);
|
|
5741
|
+
index += 1;
|
|
5742
|
+
}
|
|
5743
|
+
while (index < length && text[index] === " ") index += 1;
|
|
5744
|
+
let word = "";
|
|
5745
|
+
while (index < length && /[a-z]/i.test(text[index])) {
|
|
5746
|
+
word += text[index].toLowerCase();
|
|
5747
|
+
index += 1;
|
|
5748
|
+
}
|
|
5749
|
+
const unitMs = DURATION_UNITS[word];
|
|
5750
|
+
if (unitMs !== void 0) total += value * unitMs;
|
|
5751
|
+
}
|
|
5752
|
+
return total;
|
|
5753
|
+
}
|
|
5754
|
+
__name(parseDuration, "parseDuration");
|
|
5755
|
+
var DURATION_UNITS = {
|
|
5756
|
+
d: 864e5,
|
|
5757
|
+
day: 864e5,
|
|
5758
|
+
days: 864e5,
|
|
5759
|
+
h: 36e5,
|
|
5760
|
+
hr: 36e5,
|
|
5761
|
+
hrs: 36e5,
|
|
5762
|
+
hour: 36e5,
|
|
5763
|
+
hours: 36e5,
|
|
5764
|
+
m: 6e4,
|
|
5765
|
+
min: 6e4,
|
|
5766
|
+
mins: 6e4,
|
|
5767
|
+
minute: 6e4,
|
|
5768
|
+
minutes: 6e4,
|
|
5769
|
+
s: 1e3,
|
|
5770
|
+
sec: 1e3,
|
|
5771
|
+
secs: 1e3,
|
|
5772
|
+
second: 1e3,
|
|
5773
|
+
seconds: 1e3
|
|
5774
|
+
};
|
|
5775
|
+
function isDigit(code) {
|
|
5776
|
+
return code >= 48 && code <= 57;
|
|
5777
|
+
}
|
|
5778
|
+
__name(isDigit, "isDigit");
|
|
5779
|
+
function resolveZone(zone) {
|
|
5780
|
+
if (typeof zone !== "string" || !zone.trim()) return null;
|
|
5781
|
+
const candidate = zone.trim();
|
|
5782
|
+
try {
|
|
5783
|
+
new Intl.DateTimeFormat("en-US", { timeZone: candidate });
|
|
5784
|
+
return candidate;
|
|
5785
|
+
} catch {
|
|
5786
|
+
return null;
|
|
5787
|
+
}
|
|
5788
|
+
}
|
|
5789
|
+
__name(resolveZone, "resolveZone");
|
|
5790
|
+
function zonedParts(instantMs, zone) {
|
|
5791
|
+
const parts = new Intl.DateTimeFormat("en-US", {
|
|
5792
|
+
timeZone: zone,
|
|
5793
|
+
hourCycle: "h23",
|
|
5794
|
+
year: "numeric",
|
|
5795
|
+
month: "2-digit",
|
|
5796
|
+
day: "2-digit",
|
|
5797
|
+
hour: "2-digit",
|
|
5798
|
+
minute: "2-digit",
|
|
5799
|
+
second: "2-digit"
|
|
5800
|
+
}).formatToParts(new Date(instantMs));
|
|
5801
|
+
const get = /* @__PURE__ */ __name((type) => Number(parts.find((part) => part.type === type)?.value ?? 0), "get");
|
|
5802
|
+
return {
|
|
5803
|
+
year: get("year"),
|
|
5804
|
+
month: get("month") - 1,
|
|
5805
|
+
day: get("day"),
|
|
5806
|
+
hour: get("hour") % 24,
|
|
5807
|
+
minute: get("minute"),
|
|
5808
|
+
second: get("second")
|
|
5809
|
+
};
|
|
5810
|
+
}
|
|
5811
|
+
__name(zonedParts, "zonedParts");
|
|
5812
|
+
function zoneOffsetMs(instantMs, zone) {
|
|
5813
|
+
const whole = Math.floor(instantMs / 1e3) * 1e3;
|
|
5814
|
+
const p = zonedParts(whole, zone);
|
|
5815
|
+
return Date.UTC(p.year, p.month, p.day, p.hour, p.minute, p.second) - whole;
|
|
5816
|
+
}
|
|
5817
|
+
__name(zoneOffsetMs, "zoneOffsetMs");
|
|
5818
|
+
function zonedWallTimeToUtc(year, month, day, hour, minute, zone) {
|
|
5819
|
+
const asIfUtc = Date.UTC(year, month, day, hour, minute, 0);
|
|
5820
|
+
const firstOffset = zoneOffsetMs(asIfUtc, zone);
|
|
5821
|
+
const guess = asIfUtc - firstOffset;
|
|
5822
|
+
const secondOffset = zoneOffsetMs(guess, zone);
|
|
5823
|
+
return secondOffset === firstOffset ? guess : asIfUtc - secondOffset;
|
|
5824
|
+
}
|
|
5825
|
+
__name(zonedWallTimeToUtc, "zonedWallTimeToUtc");
|
|
5518
5826
|
var FLEET_ENROLLABLE_NODE_KINDS = ["desktop-node", "node"];
|
|
5519
5827
|
function isFleetEnrollableNodeKind(value) {
|
|
5520
5828
|
return typeof value === "string" && FLEET_ENROLLABLE_NODE_KINDS.includes(value);
|
|
@@ -6120,6 +6428,8 @@ var EVER_STATS_KEY_ID_HEADER = CONSTANTS.stats_headers.key_id;
|
|
|
6120
6428
|
var EVER_STATS_SIGNATURE_PREFIX = CONSTANTS.stats_signature_prefix;
|
|
6121
6429
|
var EVER_STATS_MAX_BODY_BYTES = CONSTANTS.stats.max_bytes;
|
|
6122
6430
|
var DOMAIN_EVENT_CATALOG = [
|
|
6431
|
+
// EW-P2-6 — `org:write` (and the app) create Organizations.
|
|
6432
|
+
{ type: "organization.created", domain: "organization", aggregateType: "organization", phase: "P2" },
|
|
6123
6433
|
{ type: "organization.updated", domain: "organization", aggregateType: "organization", phase: "P1" },
|
|
6124
6434
|
{
|
|
6125
6435
|
type: "organization.member.added",
|
|
@@ -6166,10 +6476,10 @@ var DOMAIN_EVENT_CATALOG = [
|
|
|
6166
6476
|
{ type: "mission.updated", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6167
6477
|
{ type: "mission.status_changed", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6168
6478
|
{ type: "mission.deleted", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6169
|
-
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6170
|
-
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6171
|
-
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6172
|
-
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
6479
|
+
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
6480
|
+
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
6481
|
+
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
6482
|
+
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
6173
6483
|
{ type: "goal.created", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
6174
6484
|
{ type: "goal.updated", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
6175
6485
|
{ type: "goal.status_changed", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
@@ -6226,6 +6536,11 @@ var DOMAIN_EVENT_TYPES = DOMAIN_EVENT_CATALOG.map((entry) => entry.type);
|
|
|
6226
6536
|
var CATALOG_BY_TYPE = new Map(
|
|
6227
6537
|
DOMAIN_EVENT_CATALOG.map((entry) => [entry.type, entry])
|
|
6228
6538
|
);
|
|
6539
|
+
var RECIPIENT_SCOPED_EVENT_DOMAINS = {
|
|
6540
|
+
inbox: "inbox:read",
|
|
6541
|
+
email: "email:read"
|
|
6542
|
+
};
|
|
6543
|
+
var RECIPIENT_EVENT_SCOPES = Object.values(RECIPIENT_SCOPED_EVENT_DOMAINS);
|
|
6229
6544
|
var INTEGRATION_SCOPES = [
|
|
6230
6545
|
"org:read",
|
|
6231
6546
|
"org:write",
|
|
@@ -7371,237 +7686,574 @@ function readPendingComputerSessions(payload) {
|
|
|
7371
7686
|
}
|
|
7372
7687
|
__name(readPendingComputerSessions, "readPendingComputerSessions");
|
|
7373
7688
|
|
|
7374
|
-
// src/core/
|
|
7375
|
-
var
|
|
7376
|
-
|
|
7377
|
-
|
|
7378
|
-
|
|
7689
|
+
// src/core/heartbeat.ts
|
|
7690
|
+
var systemScheduler = {
|
|
7691
|
+
setTimeout: /* @__PURE__ */ __name((callback, ms) => setTimeout(callback, ms), "setTimeout"),
|
|
7692
|
+
clearTimeout: /* @__PURE__ */ __name((handle) => clearTimeout(handle), "clearTimeout")
|
|
7693
|
+
};
|
|
7694
|
+
var OPTIONAL_DESCRIPTION_FIELD_TIERS = [
|
|
7695
|
+
// NEWEST first. Node lifecycle + remote node limits (self-build slices
|
|
7696
|
+
// AR, AS). Tiered (review): `cliVersions` goes out on EVERY beat of a
|
|
7697
|
+
// node with a pinned CLI, so an upgraded node talking to a platform that
|
|
7698
|
+
// predates it would otherwise drop the worker state and housekeeping
|
|
7699
|
+
// along with it — losing health signals that platform DOES understand,
|
|
7700
|
+
// for the life of the process. Stripping the newest tier first keeps them.
|
|
7701
|
+
["cliVersions", "maxConcurrentJobs", "maxCpuPercent", "maxMemoryMb"],
|
|
7702
|
+
// Fleet health signals (EW-776) and node housekeeping (EW-803).
|
|
7703
|
+
[
|
|
7704
|
+
"workerState",
|
|
7705
|
+
"workerStateReason",
|
|
7706
|
+
"minFreeDiskBytes",
|
|
7707
|
+
"workspaceCount",
|
|
7708
|
+
"workspaceBytes",
|
|
7709
|
+
"lastReclaimAt",
|
|
7710
|
+
"lastReclaimFreedBytes"
|
|
7711
|
+
]
|
|
7712
|
+
];
|
|
7713
|
+
function untilHeartbeatAccepted(loop, shouldWait = () => true) {
|
|
7714
|
+
const verdict = /* @__PURE__ */ __name((state) => {
|
|
7715
|
+
if (state.state === "connected") return true;
|
|
7716
|
+
if (state.state === "stopped" || !shouldWait()) return false;
|
|
7379
7717
|
return null;
|
|
7380
|
-
}
|
|
7381
|
-
const
|
|
7382
|
-
|
|
7718
|
+
}, "verdict");
|
|
7719
|
+
const now = verdict(loop.getState());
|
|
7720
|
+
if (now !== null) return Promise.resolve(now);
|
|
7721
|
+
return new Promise((resolve9) => {
|
|
7722
|
+
const unsubscribe = loop.onChange((state) => {
|
|
7723
|
+
const settled = verdict(state);
|
|
7724
|
+
if (settled === null) return;
|
|
7725
|
+
unsubscribe();
|
|
7726
|
+
resolve9(settled);
|
|
7727
|
+
});
|
|
7728
|
+
});
|
|
7383
7729
|
}
|
|
7384
|
-
__name(
|
|
7385
|
-
function
|
|
7386
|
-
|
|
7387
|
-
|
|
7730
|
+
__name(untilHeartbeatAccepted, "untilHeartbeatAccepted");
|
|
7731
|
+
function computeBackoffDelay(intervalMs, failures, maxMs) {
|
|
7732
|
+
if (failures <= 0) {
|
|
7733
|
+
return intervalMs;
|
|
7734
|
+
}
|
|
7735
|
+
const exponent = Math.min(failures, 30);
|
|
7736
|
+
const delay2 = intervalMs * 2 ** exponent;
|
|
7737
|
+
return Math.min(delay2, maxMs);
|
|
7388
7738
|
}
|
|
7389
|
-
__name(
|
|
7390
|
-
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
if (pinnedTo === null) {
|
|
7394
|
-
return { url: configuredUrl, source: "config", configuredUrl, pinnedTo: null, mismatch: false };
|
|
7739
|
+
__name(computeBackoffDelay, "computeBackoffDelay");
|
|
7740
|
+
var HeartbeatLoop = class {
|
|
7741
|
+
static {
|
|
7742
|
+
__name(this, "HeartbeatLoop");
|
|
7395
7743
|
}
|
|
7396
|
-
|
|
7397
|
-
|
|
7398
|
-
|
|
7399
|
-
|
|
7400
|
-
|
|
7401
|
-
|
|
7744
|
+
options;
|
|
7745
|
+
scheduler;
|
|
7746
|
+
now;
|
|
7747
|
+
listeners = /* @__PURE__ */ new Set();
|
|
7748
|
+
timer = null;
|
|
7749
|
+
running = false;
|
|
7750
|
+
inFlight = null;
|
|
7751
|
+
/**
|
|
7752
|
+
* How many optional-field TIERS (newest first) this platform has proven
|
|
7753
|
+
* it predates; from then on the beat leaves those tiers out. 0 = it
|
|
7754
|
+
* takes everything. Process-scoped on purpose — a platform upgrade is a
|
|
7755
|
+
* restart-shaped event for the node, and re-probing on every beat would
|
|
7756
|
+
* mean one wasted round trip per beat, forever, against an API that
|
|
7757
|
+
* will never accept them.
|
|
7758
|
+
*/
|
|
7759
|
+
strippedTiers = 0;
|
|
7760
|
+
state = {
|
|
7761
|
+
state: "idle",
|
|
7762
|
+
lastHeartbeatAt: null,
|
|
7763
|
+
consecutiveFailures: 0,
|
|
7764
|
+
nextAttemptInMs: null,
|
|
7765
|
+
lastError: null,
|
|
7766
|
+
lastErrorKind: null,
|
|
7767
|
+
node: null
|
|
7402
7768
|
};
|
|
7403
|
-
|
|
7404
|
-
|
|
7405
|
-
|
|
7406
|
-
|
|
7407
|
-
|
|
7769
|
+
constructor(options) {
|
|
7770
|
+
this.options = {
|
|
7771
|
+
...options,
|
|
7772
|
+
intervalMs: options.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS,
|
|
7773
|
+
maxBackoffMs: options.maxBackoffMs ?? MAX_HEARTBEAT_BACKOFF_MS
|
|
7774
|
+
};
|
|
7775
|
+
this.scheduler = options.scheduler ?? systemScheduler;
|
|
7776
|
+
this.now = options.now ?? (() => Date.now());
|
|
7777
|
+
this.options.logger?.protect(options.secret);
|
|
7408
7778
|
}
|
|
7409
|
-
|
|
7410
|
-
return
|
|
7779
|
+
getState() {
|
|
7780
|
+
return { ...this.state };
|
|
7411
7781
|
}
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
|
|
7418
|
-
|
|
7419
|
-
|
|
7420
|
-
|
|
7421
|
-
|
|
7422
|
-
if (typeof stdout !== "string") return null;
|
|
7423
|
-
const match = /\b(\d+\.\d+(?:\.\d+)?(?:[-+][0-9A-Za-z.-]+)?)\b/.exec(stdout);
|
|
7424
|
-
return match ? match[1] : null;
|
|
7425
|
-
}
|
|
7426
|
-
__name(parseCliVersion, "parseCliVersion");
|
|
7427
|
-
async function detectAgentCliVersion(runner, candidates = AGENT_CLI_CANDIDATES) {
|
|
7428
|
-
for (const command of candidates) {
|
|
7429
|
-
try {
|
|
7430
|
-
const result = await runner.run(command, ["--version"]);
|
|
7431
|
-
if (result.code !== 0) continue;
|
|
7432
|
-
const version = parseCliVersion(`${result.stdout ?? ""}
|
|
7433
|
-
${result.stderr ?? ""}`);
|
|
7434
|
-
if (!version) continue;
|
|
7435
|
-
return `${command} ${version}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH);
|
|
7436
|
-
} catch {
|
|
7437
|
-
continue;
|
|
7782
|
+
onChange(listener) {
|
|
7783
|
+
this.listeners.add(listener);
|
|
7784
|
+
return () => {
|
|
7785
|
+
this.listeners.delete(listener);
|
|
7786
|
+
};
|
|
7787
|
+
}
|
|
7788
|
+
/** Start beating. Resolves once the FIRST beat has settled. */
|
|
7789
|
+
start() {
|
|
7790
|
+
if (this.running) {
|
|
7791
|
+
return this.inFlight ?? Promise.resolve();
|
|
7438
7792
|
}
|
|
7793
|
+
this.running = true;
|
|
7794
|
+
return this.tick();
|
|
7439
7795
|
}
|
|
7440
|
-
|
|
7441
|
-
|
|
7442
|
-
|
|
7443
|
-
|
|
7444
|
-
|
|
7445
|
-
const free = await io.freeBytes(path2);
|
|
7446
|
-
if (typeof free !== "number" || !Number.isFinite(free) || free < 0) return null;
|
|
7447
|
-
return Math.floor(free);
|
|
7448
|
-
} catch {
|
|
7449
|
-
return null;
|
|
7796
|
+
/** Stop beating and cancel any pending timer. */
|
|
7797
|
+
stop() {
|
|
7798
|
+
this.running = false;
|
|
7799
|
+
this.cancelTimer();
|
|
7800
|
+
this.patch({ state: "stopped", nextAttemptInMs: null });
|
|
7450
7801
|
}
|
|
7451
|
-
|
|
7452
|
-
|
|
7453
|
-
|
|
7454
|
-
function parseClaudeAuthStatus(stdout) {
|
|
7455
|
-
let parsed;
|
|
7456
|
-
try {
|
|
7457
|
-
parsed = JSON.parse(stdout.trim());
|
|
7458
|
-
} catch {
|
|
7459
|
-
return null;
|
|
7802
|
+
/** Await the in-flight beat, if any (test/shutdown helper). */
|
|
7803
|
+
async settled() {
|
|
7804
|
+
await this.inFlight;
|
|
7460
7805
|
}
|
|
7461
|
-
|
|
7462
|
-
|
|
7463
|
-
|
|
7464
|
-
|
|
7465
|
-
|
|
7466
|
-
|
|
7467
|
-
|
|
7468
|
-
|
|
7469
|
-
|
|
7470
|
-
|
|
7471
|
-
|
|
7472
|
-
|
|
7473
|
-
|
|
7474
|
-
|
|
7475
|
-
|
|
7476
|
-
|
|
7477
|
-
|
|
7478
|
-
|
|
7479
|
-
|
|
7480
|
-
|
|
7481
|
-
|
|
7482
|
-
|
|
7483
|
-
|
|
7484
|
-
if (typeof value !== "string") return null;
|
|
7485
|
-
const cleaned = value.replace(/[\u0000-\u001F\u007F]+/g, " ").trim();
|
|
7486
|
-
return cleaned ? cleaned : null;
|
|
7487
|
-
}
|
|
7488
|
-
__name(compactString, "compactString");
|
|
7489
|
-
async function detectModelIdentity(runner, paths = {}) {
|
|
7490
|
-
const candidates = [
|
|
7491
|
-
{ command: paths["claude-code"] || "claude", args: ["auth", "status", "--json"], parse: parseClaudeAuthStatus },
|
|
7492
|
-
{ command: paths.codex || "codex", args: ["login", "status"], parse: parseCodexLoginStatus }
|
|
7493
|
-
];
|
|
7494
|
-
for (const candidate of candidates) {
|
|
7495
|
-
try {
|
|
7496
|
-
const result = await runner.run(candidate.command, candidate.args);
|
|
7497
|
-
const stdout = result.stdout ?? "";
|
|
7498
|
-
const stderr = result.stderr ?? "";
|
|
7499
|
-
const label = candidate.parse(stdout) ?? candidate.parse(`${stdout}
|
|
7500
|
-
${stderr}`);
|
|
7501
|
-
if (label) return label.slice(0, FLEET_MAX_MODEL_IDENTITY_LENGTH);
|
|
7502
|
-
} catch {
|
|
7503
|
-
continue;
|
|
7806
|
+
/** Run one beat now. Exposed so the UI can offer "retry" and tests can step. */
|
|
7807
|
+
tick() {
|
|
7808
|
+
const run = this.runOnce().finally(() => {
|
|
7809
|
+
if (this.inFlight === run) {
|
|
7810
|
+
this.inFlight = null;
|
|
7811
|
+
}
|
|
7812
|
+
});
|
|
7813
|
+
this.inFlight = run;
|
|
7814
|
+
return run;
|
|
7815
|
+
}
|
|
7816
|
+
async runOnce() {
|
|
7817
|
+
if (!this.running) {
|
|
7818
|
+
return;
|
|
7819
|
+
}
|
|
7820
|
+
this.cancelTimer();
|
|
7821
|
+
if (this.state.state !== "connected") {
|
|
7822
|
+
this.patch({ state: "connecting" });
|
|
7823
|
+
}
|
|
7824
|
+
try {
|
|
7825
|
+
const description = await this.options.describe();
|
|
7826
|
+
this.onSuccess(await this.beat(description));
|
|
7827
|
+
} catch (error) {
|
|
7828
|
+
this.onFailure(error);
|
|
7504
7829
|
}
|
|
7830
|
+
this.scheduleNext();
|
|
7505
7831
|
}
|
|
7506
|
-
|
|
7507
|
-
|
|
7508
|
-
|
|
7509
|
-
|
|
7510
|
-
|
|
7511
|
-
|
|
7512
|
-
|
|
7513
|
-
|
|
7514
|
-
|
|
7515
|
-
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
|
|
7521
|
-
|
|
7832
|
+
/**
|
|
7833
|
+
* Send one beat, tolerating a platform that predates the optional
|
|
7834
|
+
* self-description fields (the worker state, EW-776; the housekeeping
|
|
7835
|
+
* report, EW-803).
|
|
7836
|
+
*
|
|
7837
|
+
* The API validates heartbeats with `whitelist + forbidNonWhitelisted`,
|
|
7838
|
+
* so a field an older build does not know is not ignored — it 400s the
|
|
7839
|
+
* whole request. A 400 is a FAILED beat, a failed beat backs off, and
|
|
7840
|
+
* enough of them sweep the node to `offline`. Which would mean shipping
|
|
7841
|
+
* a health-reporting feature whose failure mode is "every node in a
|
|
7842
|
+
* mixed-version fleet goes dark": the exact class of outage this slice
|
|
7843
|
+
* was written to end.
|
|
7844
|
+
*
|
|
7845
|
+
* So a 400 on a beat that CARRIED those fields is retried once,
|
|
7846
|
+
* immediately, without them. If that succeeds the platform is simply
|
|
7847
|
+
* older than this daemon; we latch, say so once, and keep reporting
|
|
7848
|
+
* liveness. The retry costs one request, once per process.
|
|
7849
|
+
*
|
|
7850
|
+
* Inferring "the field was rejected" from a bare 400 is deliberate and
|
|
7851
|
+
* has a cost worth stating: `FleetClient` never surfaces server bodies
|
|
7852
|
+
* (fixed client-authored messages only), so a 400 caused by something
|
|
7853
|
+
* else on such a beat also latches, and this node reports liveness only
|
|
7854
|
+
* until it restarts. That is the safe direction — the node stays
|
|
7855
|
+
* observable either way, and only the extra signal is lost.
|
|
7856
|
+
*/
|
|
7857
|
+
async beat(description) {
|
|
7858
|
+
const credential = { nodeId: this.options.nodeId, secret: this.options.secret };
|
|
7859
|
+
let sent = stripTiers(description, this.strippedTiers);
|
|
7860
|
+
try {
|
|
7861
|
+
return await this.options.client.heartbeat({ ...credential, ...sent });
|
|
7862
|
+
} catch (error) {
|
|
7863
|
+
if (!isFieldRejection(error)) throw error;
|
|
7864
|
+
let lastError = error;
|
|
7865
|
+
for (let tiers = this.strippedTiers + 1; tiers <= OPTIONAL_DESCRIPTION_FIELD_TIERS.length; tiers += 1) {
|
|
7866
|
+
const candidate = stripTiers(description, tiers);
|
|
7867
|
+
if (Object.keys(candidate).length === Object.keys(sent).length) continue;
|
|
7868
|
+
sent = candidate;
|
|
7869
|
+
try {
|
|
7870
|
+
const result = await this.options.client.heartbeat({ ...credential, ...candidate });
|
|
7871
|
+
this.strippedTiers = tiers;
|
|
7872
|
+
this.options.logger?.warn(
|
|
7873
|
+
tiers >= OPTIONAL_DESCRIPTION_FIELD_TIERS.length ? "Platform rejected the optional self-description fields (worker state, housekeeping); it predates them. Reporting liveness only until this node restarts." : "Platform rejected the newest optional self-description fields (pinned CLI versions, enforced limits); it predates them. Still reporting worker state and housekeeping; the newer fields stay off until this node restarts."
|
|
7874
|
+
);
|
|
7875
|
+
return result;
|
|
7876
|
+
} catch (retryError) {
|
|
7877
|
+
if (!isFieldRejection(retryError)) throw retryError;
|
|
7878
|
+
lastError = retryError;
|
|
7879
|
+
}
|
|
7880
|
+
}
|
|
7881
|
+
throw lastError;
|
|
7882
|
+
}
|
|
7883
|
+
}
|
|
7884
|
+
onSuccess(result) {
|
|
7885
|
+
this.patch({
|
|
7886
|
+
state: "connected",
|
|
7887
|
+
lastHeartbeatAt: this.now(),
|
|
7888
|
+
consecutiveFailures: 0,
|
|
7889
|
+
lastError: null,
|
|
7890
|
+
lastErrorKind: null,
|
|
7891
|
+
node: result.node
|
|
7522
7892
|
});
|
|
7523
|
-
|
|
7524
|
-
|
|
7525
|
-
|
|
7526
|
-
|
|
7527
|
-
|
|
7528
|
-
|
|
7529
|
-
var MODEL_CLI_EMPTY_MCP_CONFIG = `${JSON.stringify({ mcpServers: {} }, null, 2)}
|
|
7530
|
-
`;
|
|
7531
|
-
function modelCliMcpIsolationEnv(provider) {
|
|
7532
|
-
return provider === "claude-code" ? { ENABLE_CLAUDEAI_MCP_SERVERS: "false" } : null;
|
|
7533
|
-
}
|
|
7534
|
-
__name(modelCliMcpIsolationEnv, "modelCliMcpIsolationEnv");
|
|
7535
|
-
function claudeManagedMcpConfigPath(platform = process.platform) {
|
|
7536
|
-
if (platform === "win32") return "C:\\Program Files\\ClaudeCode\\managed-mcp.json";
|
|
7537
|
-
if (platform === "darwin") return "/Library/Application Support/ClaudeCode/managed-mcp.json";
|
|
7538
|
-
return "/etc/claude-code/managed-mcp.json";
|
|
7539
|
-
}
|
|
7540
|
-
__name(claudeManagedMcpConfigPath, "claudeManagedMcpConfigPath");
|
|
7541
|
-
function claudeManagedMcpConfigInEffect(raw) {
|
|
7542
|
-
if (typeof raw !== "string" || !raw.trim()) return false;
|
|
7543
|
-
try {
|
|
7544
|
-
const parsed = JSON.parse(raw);
|
|
7545
|
-
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed);
|
|
7546
|
-
} catch {
|
|
7547
|
-
return false;
|
|
7893
|
+
if (this.options.onAccepted) {
|
|
7894
|
+
try {
|
|
7895
|
+
this.options.onAccepted(result);
|
|
7896
|
+
} catch {
|
|
7897
|
+
}
|
|
7898
|
+
}
|
|
7548
7899
|
}
|
|
7549
|
-
|
|
7550
|
-
|
|
7551
|
-
|
|
7552
|
-
|
|
7553
|
-
|
|
7900
|
+
onFailure(error) {
|
|
7901
|
+
const kind = error instanceof FleetClientError ? error.kind : "network";
|
|
7902
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
7903
|
+
const message = this.options.logger?.redact(raw) ?? raw;
|
|
7904
|
+
const failures = this.state.consecutiveFailures + 1;
|
|
7905
|
+
this.patch({
|
|
7906
|
+
// `unauthorized` is sticky and visible: the operator has to act
|
|
7907
|
+
// (re-enable the node in Fleet, or re-enroll). We keep retrying at
|
|
7908
|
+
// the backoff ceiling so a re-enabled node recovers on its own.
|
|
7909
|
+
state: kind === "unauthorized" ? "unauthorized" : "retrying",
|
|
7910
|
+
consecutiveFailures: failures,
|
|
7911
|
+
lastError: message,
|
|
7912
|
+
lastErrorKind: kind
|
|
7913
|
+
});
|
|
7914
|
+
this.options.logger?.warn(`Heartbeat failed (attempt ${failures}): ${message}`);
|
|
7554
7915
|
}
|
|
7555
|
-
|
|
7556
|
-
|
|
7557
|
-
|
|
7916
|
+
scheduleNext() {
|
|
7917
|
+
if (!this.running) {
|
|
7918
|
+
return;
|
|
7919
|
+
}
|
|
7920
|
+
const delay2 = computeBackoffDelay(
|
|
7921
|
+
this.options.intervalMs,
|
|
7922
|
+
this.state.consecutiveFailures,
|
|
7923
|
+
this.options.maxBackoffMs
|
|
7924
|
+
);
|
|
7925
|
+
this.patch({ nextAttemptInMs: delay2 });
|
|
7926
|
+
this.timer = this.scheduler.setTimeout(() => {
|
|
7927
|
+
this.timer = null;
|
|
7928
|
+
void this.tick();
|
|
7929
|
+
}, delay2);
|
|
7558
7930
|
}
|
|
7559
|
-
|
|
7560
|
-
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
7564
|
-
var UNSAFE_PATH_CHARS = /[\0\r\n"'`$&|;<>%!^]/;
|
|
7565
|
-
function quoteShellPath(path2, platform = process.platform) {
|
|
7566
|
-
if (typeof path2 !== "string" || !path2.trim()) {
|
|
7567
|
-
throw new ModelCliCommandError("Path must not be empty");
|
|
7931
|
+
cancelTimer() {
|
|
7932
|
+
if (this.timer !== null) {
|
|
7933
|
+
this.scheduler.clearTimeout(this.timer);
|
|
7934
|
+
this.timer = null;
|
|
7935
|
+
}
|
|
7568
7936
|
}
|
|
7569
|
-
|
|
7570
|
-
|
|
7937
|
+
patch(next) {
|
|
7938
|
+
this.state = { ...this.state, ...next };
|
|
7939
|
+
const snapshot = this.getState();
|
|
7940
|
+
for (const listener of this.listeners) {
|
|
7941
|
+
listener(snapshot);
|
|
7942
|
+
}
|
|
7571
7943
|
}
|
|
7572
|
-
|
|
7573
|
-
|
|
7574
|
-
|
|
7944
|
+
};
|
|
7945
|
+
function stripTiers(description, tiers) {
|
|
7946
|
+
const out = {};
|
|
7947
|
+
const dropped = new Set(OPTIONAL_DESCRIPTION_FIELD_TIERS.slice(0, tiers).flat());
|
|
7948
|
+
for (const [key, value] of Object.entries(description)) {
|
|
7949
|
+
if (value === void 0 || dropped.has(key)) continue;
|
|
7950
|
+
out[key] = value;
|
|
7575
7951
|
}
|
|
7576
|
-
return
|
|
7952
|
+
return out;
|
|
7577
7953
|
}
|
|
7578
|
-
__name(
|
|
7579
|
-
function
|
|
7580
|
-
|
|
7581
|
-
throw new ModelCliCommandError("MCP server name is not an opaque identifier");
|
|
7582
|
-
}
|
|
7583
|
-
return name;
|
|
7954
|
+
__name(stripTiers, "stripTiers");
|
|
7955
|
+
function isFieldRejection(error) {
|
|
7956
|
+
return error instanceof FleetClientError && error.kind === "invalid-request" && error.status === 400;
|
|
7584
7957
|
}
|
|
7585
|
-
__name(
|
|
7586
|
-
|
|
7587
|
-
|
|
7588
|
-
|
|
7958
|
+
__name(isFieldRejection, "isFieldRejection");
|
|
7959
|
+
|
|
7960
|
+
// src/core/api-base.ts
|
|
7961
|
+
var API_URL_ENV = "EVER_WORKS_NODE_API_URL";
|
|
7962
|
+
function rawPin(env) {
|
|
7963
|
+
const value = env[API_URL_ENV];
|
|
7964
|
+
if (typeof value !== "string") {
|
|
7965
|
+
return null;
|
|
7589
7966
|
}
|
|
7590
|
-
|
|
7967
|
+
const trimmed = value.trim();
|
|
7968
|
+
return trimmed ? trimmed : null;
|
|
7591
7969
|
}
|
|
7592
|
-
__name(
|
|
7593
|
-
|
|
7594
|
-
|
|
7595
|
-
|
|
7596
|
-
|
|
7597
|
-
|
|
7598
|
-
|
|
7970
|
+
__name(rawPin, "rawPin");
|
|
7971
|
+
function readApiUrlPin(env = process.env) {
|
|
7972
|
+
const raw = rawPin(env);
|
|
7973
|
+
return raw === null ? null : normalizeApiUrl(raw);
|
|
7974
|
+
}
|
|
7975
|
+
__name(readApiUrlPin, "readApiUrlPin");
|
|
7976
|
+
function resolveApiBase(config, env = process.env) {
|
|
7977
|
+
const configuredUrl = config.apiUrl;
|
|
7978
|
+
const pinnedTo = readApiUrlPin(env);
|
|
7979
|
+
if (pinnedTo === null) {
|
|
7980
|
+
return { url: configuredUrl, source: "config", configuredUrl, pinnedTo: null, mismatch: false };
|
|
7599
7981
|
}
|
|
7600
|
-
return
|
|
7982
|
+
return {
|
|
7983
|
+
url: pinnedTo,
|
|
7984
|
+
source: "pin",
|
|
7985
|
+
configuredUrl,
|
|
7986
|
+
pinnedTo,
|
|
7987
|
+
mismatch: pinnedTo !== configuredUrl
|
|
7988
|
+
};
|
|
7601
7989
|
}
|
|
7602
|
-
__name(
|
|
7603
|
-
|
|
7604
|
-
|
|
7990
|
+
__name(resolveApiBase, "resolveApiBase");
|
|
7991
|
+
function describeApiBase(base) {
|
|
7992
|
+
if (base.source === "config") {
|
|
7993
|
+
return `${base.url} (from the enrolled config)`;
|
|
7994
|
+
}
|
|
7995
|
+
if (!base.mismatch) {
|
|
7996
|
+
return `${base.url} (pinned via ${API_URL_ENV}; matches the enrolled origin)`;
|
|
7997
|
+
}
|
|
7998
|
+
return `${base.url} (PINNED via ${API_URL_ENV}) \u2014 enrolled against ${base.configuredUrl}. The pin does not match the enrolled origin: this node has no credential on the pinned platform, so every call will be refused with 401 until the pin is corrected or the node is re-enrolled.`;
|
|
7999
|
+
}
|
|
8000
|
+
__name(describeApiBase, "describeApiBase");
|
|
8001
|
+
|
|
8002
|
+
// src/core/auth-client.ts
|
|
8003
|
+
var EVER_ID_DEVICE_SCOPE = ["openid", "email", EVER_ID_SCOPES.SESSION_EXCHANGE].join(" ");
|
|
8004
|
+
|
|
8005
|
+
// src/core/telemetry-probe.ts
|
|
8006
|
+
var AGENT_CLI_CANDIDATES = ["claude", "codex", "gemini", "opencode"];
|
|
8007
|
+
function parseCliVersion(stdout) {
|
|
8008
|
+
if (typeof stdout !== "string") return null;
|
|
8009
|
+
const match = /\b(\d+\.\d+(?:\.\d+)?(?:[-+][0-9A-Za-z.-]+)?)\b/.exec(stdout);
|
|
8010
|
+
return match ? match[1] : null;
|
|
8011
|
+
}
|
|
8012
|
+
__name(parseCliVersion, "parseCliVersion");
|
|
8013
|
+
async function detectAgentCliVersion(runner, candidates = AGENT_CLI_CANDIDATES) {
|
|
8014
|
+
for (const command of candidates) {
|
|
8015
|
+
try {
|
|
8016
|
+
const result = await runner.run(command, ["--version"]);
|
|
8017
|
+
if (result.code !== 0) continue;
|
|
8018
|
+
const version = parseCliVersion(`${result.stdout ?? ""}
|
|
8019
|
+
${result.stderr ?? ""}`);
|
|
8020
|
+
if (!version) continue;
|
|
8021
|
+
return `${command} ${version}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH);
|
|
8022
|
+
} catch {
|
|
8023
|
+
continue;
|
|
8024
|
+
}
|
|
8025
|
+
}
|
|
8026
|
+
return null;
|
|
8027
|
+
}
|
|
8028
|
+
__name(detectAgentCliVersion, "detectAgentCliVersion");
|
|
8029
|
+
async function detectDiskFreeBytes(io, path2) {
|
|
8030
|
+
try {
|
|
8031
|
+
const free = await io.freeBytes(path2);
|
|
8032
|
+
if (typeof free !== "number" || !Number.isFinite(free) || free < 0) return null;
|
|
8033
|
+
return Math.floor(free);
|
|
8034
|
+
} catch {
|
|
8035
|
+
return null;
|
|
8036
|
+
}
|
|
8037
|
+
}
|
|
8038
|
+
__name(detectDiskFreeBytes, "detectDiskFreeBytes");
|
|
8039
|
+
var MODEL_IDENTITY_CACHE_TTL_MS = 5 * 6e4;
|
|
8040
|
+
function parseClaudeAuthStatus(stdout) {
|
|
8041
|
+
let parsed;
|
|
8042
|
+
try {
|
|
8043
|
+
parsed = JSON.parse(stdout.trim());
|
|
8044
|
+
} catch {
|
|
8045
|
+
return null;
|
|
8046
|
+
}
|
|
8047
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
8048
|
+
const status = parsed;
|
|
8049
|
+
if (status.loggedIn !== true) return "claude-code: not logged in";
|
|
8050
|
+
const email = compactString(status.email);
|
|
8051
|
+
const org = compactString(status.orgName);
|
|
8052
|
+
const plan = compactString(status.subscriptionType);
|
|
8053
|
+
const method = compactString(status.authMethod);
|
|
8054
|
+
const who = email ?? (method ? `${method} login` : "logged in");
|
|
8055
|
+
const details = [org, plan].filter((part) => part !== null);
|
|
8056
|
+
return `claude-code: ${who}${details.length > 0 ? ` (${details.join(", ")})` : ""}`;
|
|
8057
|
+
}
|
|
8058
|
+
__name(parseClaudeAuthStatus, "parseClaudeAuthStatus");
|
|
8059
|
+
function parseCodexLoginStatus(stdout) {
|
|
8060
|
+
const text = stdout.trim().toLowerCase();
|
|
8061
|
+
if (!text) return null;
|
|
8062
|
+
if (text.includes("not logged in")) return "codex: not logged in";
|
|
8063
|
+
if (text.includes("chatgpt")) return "codex: chatgpt";
|
|
8064
|
+
if (text.includes("api key")) return "codex: api-key";
|
|
8065
|
+
if (text.includes("logged in")) return "codex: logged in";
|
|
8066
|
+
return null;
|
|
8067
|
+
}
|
|
8068
|
+
__name(parseCodexLoginStatus, "parseCodexLoginStatus");
|
|
8069
|
+
function compactString(value) {
|
|
8070
|
+
if (typeof value !== "string") return null;
|
|
8071
|
+
const cleaned = value.replace(/[\u0000-\u001F\u007F]+/g, " ").trim();
|
|
8072
|
+
return cleaned ? cleaned : null;
|
|
8073
|
+
}
|
|
8074
|
+
__name(compactString, "compactString");
|
|
8075
|
+
async function detectModelIdentity(runner, paths = {}) {
|
|
8076
|
+
const candidates = [
|
|
8077
|
+
{ command: paths["claude-code"] || "claude", args: ["auth", "status", "--json"], parse: parseClaudeAuthStatus },
|
|
8078
|
+
{ command: paths.codex || "codex", args: ["login", "status"], parse: parseCodexLoginStatus }
|
|
8079
|
+
];
|
|
8080
|
+
for (const candidate of candidates) {
|
|
8081
|
+
try {
|
|
8082
|
+
const result = await runner.run(candidate.command, candidate.args);
|
|
8083
|
+
const stdout = result.stdout ?? "";
|
|
8084
|
+
const stderr = result.stderr ?? "";
|
|
8085
|
+
const label = candidate.parse(stdout) ?? candidate.parse(`${stdout}
|
|
8086
|
+
${stderr}`);
|
|
8087
|
+
if (label) return label.slice(0, FLEET_MAX_MODEL_IDENTITY_LENGTH);
|
|
8088
|
+
} catch {
|
|
8089
|
+
continue;
|
|
8090
|
+
}
|
|
8091
|
+
}
|
|
8092
|
+
return null;
|
|
8093
|
+
}
|
|
8094
|
+
__name(detectModelIdentity, "detectModelIdentity");
|
|
8095
|
+
function cacheProbe(probe, ttlMs = MODEL_IDENTITY_CACHE_TTL_MS, now = Date.now) {
|
|
8096
|
+
let value = null;
|
|
8097
|
+
let expiresAt = Number.NEGATIVE_INFINITY;
|
|
8098
|
+
let inFlight = null;
|
|
8099
|
+
return async () => {
|
|
8100
|
+
if (now() < expiresAt) return value;
|
|
8101
|
+
if (inFlight) return inFlight;
|
|
8102
|
+
inFlight = probe().then((fresh) => {
|
|
8103
|
+
value = fresh;
|
|
8104
|
+
expiresAt = now() + ttlMs;
|
|
8105
|
+
return fresh;
|
|
8106
|
+
}).finally(() => {
|
|
8107
|
+
inFlight = null;
|
|
8108
|
+
});
|
|
8109
|
+
return inFlight;
|
|
8110
|
+
};
|
|
8111
|
+
}
|
|
8112
|
+
__name(cacheProbe, "cacheProbe");
|
|
8113
|
+
|
|
8114
|
+
// src/core/executors/model-cli.ts
|
|
8115
|
+
var MODEL_CLI_EMPTY_MCP_CONFIG = `${JSON.stringify({ mcpServers: {} }, null, 2)}
|
|
8116
|
+
`;
|
|
8117
|
+
var CODEX_SETTING_VALUE = /^[A-Za-z0-9._-]{1,64}$/;
|
|
8118
|
+
var TOML_STRING_ASSIGNMENT = /^([A-Za-z0-9_-]+)[ \t]*=[ \t]*(?:"([^"\\]*)"|'([^']*)')(?:[ \t]+#.*|#.*)?$/;
|
|
8119
|
+
var TOML_MULTILINE_VALUE_START = /^[^=#]*=[ \t]*("""|''')/;
|
|
8120
|
+
function readCodexUserSettings(configToml) {
|
|
8121
|
+
const settings = {};
|
|
8122
|
+
if (typeof configToml !== "string" || configToml.length === 0) return settings;
|
|
8123
|
+
let table = "";
|
|
8124
|
+
let openMultiline = null;
|
|
8125
|
+
for (const raw of configToml.split(/\r?\n/)) {
|
|
8126
|
+
if (openMultiline !== null) {
|
|
8127
|
+
if (indexOfTomlClosingDelimiter(raw, openMultiline) !== -1) openMultiline = null;
|
|
8128
|
+
continue;
|
|
8129
|
+
}
|
|
8130
|
+
const line = raw.trim();
|
|
8131
|
+
const opening = TOML_MULTILINE_VALUE_START.exec(line);
|
|
8132
|
+
if (opening) {
|
|
8133
|
+
const delimiter = opening[1];
|
|
8134
|
+
if (indexOfTomlClosingDelimiter(line.slice(opening[0].length), delimiter) === -1) {
|
|
8135
|
+
openMultiline = delimiter;
|
|
8136
|
+
continue;
|
|
8137
|
+
}
|
|
8138
|
+
}
|
|
8139
|
+
if (line.startsWith("[")) {
|
|
8140
|
+
if (line.startsWith("[[")) {
|
|
8141
|
+
table = "\0array";
|
|
8142
|
+
} else {
|
|
8143
|
+
const close = line.indexOf("]");
|
|
8144
|
+
table = close > 1 ? line.slice(1, close).trim() : "\0unknown";
|
|
8145
|
+
}
|
|
8146
|
+
continue;
|
|
8147
|
+
}
|
|
8148
|
+
const assignment = TOML_STRING_ASSIGNMENT.exec(line);
|
|
8149
|
+
if (!assignment) continue;
|
|
8150
|
+
const key = assignment[1];
|
|
8151
|
+
const value = assignment[2] ?? assignment[3] ?? "";
|
|
8152
|
+
if (!CODEX_SETTING_VALUE.test(value)) continue;
|
|
8153
|
+
if (table === "" && key === "model" && settings.model === void 0) settings.model = value;
|
|
8154
|
+
else if (table === "" && key === "model_reasoning_effort" && settings.reasoningEffort === void 0) {
|
|
8155
|
+
settings.reasoningEffort = value;
|
|
8156
|
+
} else if (table === "windows" && key === "sandbox" && settings.windowsSandbox === void 0) {
|
|
8157
|
+
settings.windowsSandbox = value;
|
|
8158
|
+
}
|
|
8159
|
+
}
|
|
8160
|
+
return settings;
|
|
8161
|
+
}
|
|
8162
|
+
__name(readCodexUserSettings, "readCodexUserSettings");
|
|
8163
|
+
function indexOfTomlClosingDelimiter(text, delimiter) {
|
|
8164
|
+
let from = 0;
|
|
8165
|
+
for (; ; ) {
|
|
8166
|
+
const at = text.indexOf(delimiter, from);
|
|
8167
|
+
if (at === -1 || delimiter === "'''") return at;
|
|
8168
|
+
let backslashes = 0;
|
|
8169
|
+
for (let i = at - 1; i >= 0 && text[i] === "\\"; i -= 1) backslashes += 1;
|
|
8170
|
+
if (backslashes % 2 === 0) return at;
|
|
8171
|
+
from = at + 1;
|
|
8172
|
+
}
|
|
8173
|
+
}
|
|
8174
|
+
__name(indexOfTomlClosingDelimiter, "indexOfTomlClosingDelimiter");
|
|
8175
|
+
function assertCodexSetting(name, value) {
|
|
8176
|
+
if (value === void 0) return void 0;
|
|
8177
|
+
if (!CODEX_SETTING_VALUE.test(value)) {
|
|
8178
|
+
throw new ModelCliCommandError(`Refusing to forward the Codex setting '${name}': not a bare identifier`);
|
|
8179
|
+
}
|
|
8180
|
+
return value;
|
|
8181
|
+
}
|
|
8182
|
+
__name(assertCodexSetting, "assertCodexSetting");
|
|
8183
|
+
function modelCliMcpIsolationEnv(provider) {
|
|
8184
|
+
return provider === "claude-code" ? { ENABLE_CLAUDEAI_MCP_SERVERS: "false" } : null;
|
|
8185
|
+
}
|
|
8186
|
+
__name(modelCliMcpIsolationEnv, "modelCliMcpIsolationEnv");
|
|
8187
|
+
function claudeManagedMcpConfigPath(platform = process.platform) {
|
|
8188
|
+
if (platform === "win32") return "C:\\Program Files\\ClaudeCode\\managed-mcp.json";
|
|
8189
|
+
if (platform === "darwin") return "/Library/Application Support/ClaudeCode/managed-mcp.json";
|
|
8190
|
+
return "/etc/claude-code/managed-mcp.json";
|
|
8191
|
+
}
|
|
8192
|
+
__name(claudeManagedMcpConfigPath, "claudeManagedMcpConfigPath");
|
|
8193
|
+
function claudeManagedMcpConfigInEffect(raw) {
|
|
8194
|
+
if (typeof raw !== "string" || !raw.trim()) return false;
|
|
8195
|
+
try {
|
|
8196
|
+
const parsed = JSON.parse(raw);
|
|
8197
|
+
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed);
|
|
8198
|
+
} catch {
|
|
8199
|
+
return false;
|
|
8200
|
+
}
|
|
8201
|
+
}
|
|
8202
|
+
__name(claudeManagedMcpConfigInEffect, "claudeManagedMcpConfigInEffect");
|
|
8203
|
+
var ModelCliCommandError = class extends Error {
|
|
8204
|
+
static {
|
|
8205
|
+
__name(this, "ModelCliCommandError");
|
|
8206
|
+
}
|
|
8207
|
+
constructor(message) {
|
|
8208
|
+
super(message);
|
|
8209
|
+
this.name = "ModelCliCommandError";
|
|
8210
|
+
}
|
|
8211
|
+
};
|
|
8212
|
+
var MODEL_CLI_STEP_ID = "model";
|
|
8213
|
+
var MODEL_CLI_OUTPUT_TAIL_BYTES = 8 * 1024;
|
|
8214
|
+
var MODEL_CLI_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
8215
|
+
var ABSOLUTE_WIN32 = /^[A-Za-z]:[\\/]/;
|
|
8216
|
+
var UNSAFE_PATH_CHARS = /[\0\r\n"'`$&|;<>%!^]/;
|
|
8217
|
+
function quoteShellPath(path2, platform = process.platform) {
|
|
8218
|
+
if (typeof path2 !== "string" || !path2.trim()) {
|
|
8219
|
+
throw new ModelCliCommandError("Path must not be empty");
|
|
8220
|
+
}
|
|
8221
|
+
if (UNSAFE_PATH_CHARS.test(path2)) {
|
|
8222
|
+
throw new ModelCliCommandError("Path contains characters the shell could interpret");
|
|
8223
|
+
}
|
|
8224
|
+
const absolute = platform === "win32" ? ABSOLUTE_WIN32.test(path2) || path2.startsWith("\\\\") : path2.startsWith("/");
|
|
8225
|
+
if (!absolute) {
|
|
8226
|
+
throw new ModelCliCommandError("Path must be absolute");
|
|
8227
|
+
}
|
|
8228
|
+
return `"${path2}"`;
|
|
8229
|
+
}
|
|
8230
|
+
__name(quoteShellPath, "quoteShellPath");
|
|
8231
|
+
function assertMcpServerName(name) {
|
|
8232
|
+
if (typeof name !== "string" || !MCP_SERVER_NAME_PATTERN.test(name)) {
|
|
8233
|
+
throw new ModelCliCommandError("MCP server name is not an opaque identifier");
|
|
8234
|
+
}
|
|
8235
|
+
return name;
|
|
8236
|
+
}
|
|
8237
|
+
__name(assertMcpServerName, "assertMcpServerName");
|
|
8238
|
+
function assertLoopbackUrl(url) {
|
|
8239
|
+
if (typeof url !== "string" || !MCP_LOOPBACK_URL_PATTERN.test(url)) {
|
|
8240
|
+
throw new ModelCliCommandError("MCP bridge URL is not a loopback URL this node produced");
|
|
8241
|
+
}
|
|
8242
|
+
return url;
|
|
8243
|
+
}
|
|
8244
|
+
__name(assertLoopbackUrl, "assertLoopbackUrl");
|
|
8245
|
+
var MCP_SERVER_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;
|
|
8246
|
+
var MCP_LOOPBACK_URL_PATTERN = /^http:\/\/127\.0\.0\.1:\d{1,5}\/mcp\/[a-f0-9]{8,64}$/;
|
|
8247
|
+
function assertModelId(model) {
|
|
8248
|
+
if (model === void 0) return null;
|
|
8249
|
+
if (!FLEET_AGENT_EXECUTION_MODEL_PATTERN.test(model)) {
|
|
8250
|
+
throw new ModelCliCommandError("Model id is not an opaque identifier");
|
|
8251
|
+
}
|
|
8252
|
+
return model;
|
|
8253
|
+
}
|
|
8254
|
+
__name(assertModelId, "assertModelId");
|
|
8255
|
+
var MODEL_CLI_OPTIONAL_FLAGS = {
|
|
8256
|
+
"claude-code": ["--effort", "--max-budget-usd"],
|
|
7605
8257
|
codex: []
|
|
7606
8258
|
};
|
|
7607
8259
|
function unsupportedOptionalModelCliFlags(execution, supportedFlags) {
|
|
@@ -7692,7 +8344,21 @@ function buildModelCliCommand(input) {
|
|
|
7692
8344
|
if (roots.length > 0) args.push("--add-dir", ...roots);
|
|
7693
8345
|
} else if (execution.provider === "codex") {
|
|
7694
8346
|
const sandbox = permissionMode === "plan" ? "read-only" : "workspace-write";
|
|
7695
|
-
args.push(
|
|
8347
|
+
args.push(
|
|
8348
|
+
"exec",
|
|
8349
|
+
"--ignore-user-config",
|
|
8350
|
+
"--json",
|
|
8351
|
+
"--sandbox",
|
|
8352
|
+
sandbox,
|
|
8353
|
+
"-C",
|
|
8354
|
+
quoteShellPath(input.workspacePath, platform)
|
|
8355
|
+
);
|
|
8356
|
+
const kept = input.codexUserSettings ?? {};
|
|
8357
|
+
const windowsSandbox = assertCodexSetting("windows.sandbox", kept.windowsSandbox);
|
|
8358
|
+
const reasoningEffort = assertCodexSetting("model_reasoning_effort", kept.reasoningEffort);
|
|
8359
|
+
const ownerModel = assertCodexSetting("model", kept.model);
|
|
8360
|
+
if (windowsSandbox) args.push("-c", `windows.sandbox=${windowsSandbox}`);
|
|
8361
|
+
if (reasoningEffort) args.push("-c", `model_reasoning_effort=${reasoningEffort}`);
|
|
7696
8362
|
if (sandbox === "workspace-write") {
|
|
7697
8363
|
for (const root of grantedRoots(writableMounts)) args.push("--add-dir", root);
|
|
7698
8364
|
}
|
|
@@ -7702,7 +8368,8 @@ function buildModelCliCommand(input) {
|
|
|
7702
8368
|
`mcp_servers.${assertMcpServerName(mcp.serverName)}.url=${assertLoopbackUrl(mcp.serverUrl)}`
|
|
7703
8369
|
);
|
|
7704
8370
|
}
|
|
7705
|
-
|
|
8371
|
+
const codexModel = model ?? ownerModel;
|
|
8372
|
+
if (codexModel) args.push("-m", codexModel);
|
|
7706
8373
|
if (execution.skipPermissions === true) args.push("--dangerously-bypass-approvals-and-sandbox");
|
|
7707
8374
|
args.push("-");
|
|
7708
8375
|
} else {
|
|
@@ -7725,7 +8392,16 @@ function assertMountGrantsInCommand(input) {
|
|
|
7725
8392
|
}
|
|
7726
8393
|
__name(assertMountGrantsInCommand, "assertMountGrantsInCommand");
|
|
7727
8394
|
var STRICT_MCP_CONFIG_IN_COMMAND = / --mcp-config "[^"]+" --strict-mcp-config /;
|
|
8395
|
+
var CODEX_IGNORE_USER_CONFIG_IN_COMMAND = / exec --ignore-user-config /;
|
|
7728
8396
|
function assertMcpIsolationInCommand(input) {
|
|
8397
|
+
if (input.execution.provider === "codex") {
|
|
8398
|
+
if (!CODEX_IGNORE_USER_CONFIG_IN_COMMAND.test(input.command)) {
|
|
8399
|
+
throw new ModelCliCommandError(
|
|
8400
|
+
"The codex command line does not carry exec --ignore-user-config, so the run would load the machine owner's own config.toml: its MCP servers, plugins and notify hook"
|
|
8401
|
+
);
|
|
8402
|
+
}
|
|
8403
|
+
return;
|
|
8404
|
+
}
|
|
7729
8405
|
if (input.execution.provider !== "claude-code") return;
|
|
7730
8406
|
if (!STRICT_MCP_CONFIG_IN_COMMAND.test(input.command)) {
|
|
7731
8407
|
throw new ModelCliCommandError(
|
|
@@ -8053,7 +8729,7 @@ var MODEL_CLI_REQUIRED_FLAGS = {
|
|
|
8053
8729
|
// bridge's. A CLI without them cannot run fleet work at all, so `doctor`
|
|
8054
8730
|
// must say INCOMPATIBLE, not "runs that need them will fail".
|
|
8055
8731
|
"claude-code": ["-p", "--output-format", "--permission-mode", "--mcp-config", "--strict-mcp-config"],
|
|
8056
|
-
codex: ["--json", "--sandbox", "-C"]
|
|
8732
|
+
codex: ["--json", "--sandbox", "-C", "--ignore-user-config"]
|
|
8057
8733
|
};
|
|
8058
8734
|
var MODEL_CLI_CONDITIONAL_FLAGS = {
|
|
8059
8735
|
"claude-code": [
|
|
@@ -8099,390 +8775,137 @@ var ModelCliCompatibilityProbe = class {
|
|
|
8099
8775
|
const pending = this.inFlight.get(key);
|
|
8100
8776
|
if (pending) return pending;
|
|
8101
8777
|
const run = this.run(provider, executable).then(({ result, answered }) => {
|
|
8102
|
-
this.cache.set(`${provider}|${executable}`, {
|
|
8103
|
-
key,
|
|
8104
|
-
// A COMPLETE answer from a stamped file is kept until the file
|
|
8105
|
-
// changes or, at the latest, for `stampedMaxAgeMs` (an upgrade
|
|
8106
|
-
// behind a stable wrapper does not change the stamp). An
|
|
8107
|
-
// incomplete one is retried soon, and an unstamped one on the
|
|
8108
|
-
// ten-minute clock.
|
|
8109
|
-
ttlMs: !answered ? this.incompleteRetryMs : stamp !== null ? this.stampedMaxAgeMs : this.unstampedTtlMs,
|
|
8110
|
-
probedAt: this.now(),
|
|
8111
|
-
result
|
|
8112
|
-
});
|
|
8113
|
-
return result;
|
|
8114
|
-
}).finally(() => {
|
|
8115
|
-
this.inFlight.delete(key);
|
|
8116
|
-
});
|
|
8117
|
-
this.inFlight.set(key, run);
|
|
8118
|
-
return run;
|
|
8119
|
-
}
|
|
8120
|
-
/** Every pinned binary in `paths`, in provider order. Unpinned providers are skipped. */
|
|
8121
|
-
async probeAll(paths) {
|
|
8122
|
-
const out = [];
|
|
8123
|
-
for (const provider of MODEL_CLI_PROVIDERS) {
|
|
8124
|
-
const executable = paths?.[provider];
|
|
8125
|
-
if (typeof executable !== "string" || !executable.trim()) continue;
|
|
8126
|
-
out.push(await this.probe(provider, executable));
|
|
8127
|
-
}
|
|
8128
|
-
return out;
|
|
8129
|
-
}
|
|
8130
|
-
async stamp(executable) {
|
|
8131
|
-
if (!this.options.statFile) return null;
|
|
8132
|
-
try {
|
|
8133
|
-
const stamp = await this.options.statFile(executable);
|
|
8134
|
-
return stamp && Number.isFinite(stamp.mtimeMs) && Number.isFinite(stamp.size) ? stamp : null;
|
|
8135
|
-
} catch {
|
|
8136
|
-
return null;
|
|
8137
|
-
}
|
|
8138
|
-
}
|
|
8139
|
-
async run(provider, executable) {
|
|
8140
|
-
const command = this.commandFor(executable);
|
|
8141
|
-
const versionOutput = await this.ask(command, ["--version"]);
|
|
8142
|
-
const version = versionOutput === null ? null : parseCliVersion(versionOutput);
|
|
8143
|
-
const help = await this.ask(command, [...HELP_ARGS[provider]]);
|
|
8144
|
-
return {
|
|
8145
|
-
result: judgeModelCliHelp(provider, executable, version, help),
|
|
8146
|
-
// "Answered" = both invocations produced output. An UNRECOGNISED
|
|
8147
|
-
// help text is still an answer — deterministic for that file — and
|
|
8148
|
-
// re-asking every minute would learn nothing.
|
|
8149
|
-
answered: versionOutput !== null && help !== null
|
|
8150
|
-
};
|
|
8151
|
-
}
|
|
8152
|
-
/**
|
|
8153
|
-
* The runner on Windows goes through `cmd.exe` (`shell: true`), which
|
|
8154
|
-
* splits an unquoted `C:\Program Files\…` at the space.
|
|
8155
|
-
*/
|
|
8156
|
-
commandFor(executable) {
|
|
8157
|
-
return this.options.platform === "win32" && /\s/.test(executable) && !executable.startsWith('"') ? `"${executable}"` : executable;
|
|
8158
|
-
}
|
|
8159
|
-
/** stdout+stderr of one invocation, or null when it could not be run at all. */
|
|
8160
|
-
async ask(command, args) {
|
|
8161
|
-
try {
|
|
8162
|
-
const result = await this.options.runner.run(command, args);
|
|
8163
|
-
if (result.timedOut) return null;
|
|
8164
|
-
const text = `${result.stdout ?? ""}
|
|
8165
|
-
${result.stderr ?? ""}`;
|
|
8166
|
-
return result.code === 0 || text.trim() ? text : null;
|
|
8167
|
-
} catch {
|
|
8168
|
-
return null;
|
|
8169
|
-
}
|
|
8170
|
-
}
|
|
8171
|
-
};
|
|
8172
|
-
function parseAdvertisedFlags(help) {
|
|
8173
|
-
const flags = /* @__PURE__ */ new Set();
|
|
8174
|
-
const pattern = /(?:^|[\s,[(|])(--?[A-Za-z][A-Za-z0-9-]*)(?=$|[\s,=<[\])|.:])/gm;
|
|
8175
|
-
for (const match of help.matchAll(pattern)) {
|
|
8176
|
-
flags.add(match[1]);
|
|
8177
|
-
}
|
|
8178
|
-
return flags;
|
|
8179
|
-
}
|
|
8180
|
-
__name(parseAdvertisedFlags, "parseAdvertisedFlags");
|
|
8181
|
-
function judgeModelCliHelp(provider, executable, version, help) {
|
|
8182
|
-
const required = MODEL_CLI_REQUIRED_FLAGS[provider];
|
|
8183
|
-
const base = {
|
|
8184
|
-
provider,
|
|
8185
|
-
executable,
|
|
8186
|
-
version,
|
|
8187
|
-
supportedFlags: null,
|
|
8188
|
-
missingRequiredFlags: [],
|
|
8189
|
-
missingConditionalFlags: [],
|
|
8190
|
-
unsupportedOptionalFlags: [],
|
|
8191
|
-
note: null
|
|
8192
|
-
};
|
|
8193
|
-
if (help === null) {
|
|
8194
|
-
return { ...base, note: "the binary did not answer --help; nothing is dropped" };
|
|
8195
|
-
}
|
|
8196
|
-
const advertised = parseAdvertisedFlags(help);
|
|
8197
|
-
if (!required.some((flag) => advertised.has(flag))) {
|
|
8198
|
-
return {
|
|
8199
|
-
...base,
|
|
8200
|
-
note: `the help text lists none of the flags this node always passes (${required.join(", ")}); its format is not recognised, so nothing is dropped`
|
|
8201
|
-
};
|
|
8202
|
-
}
|
|
8203
|
-
const missing = /* @__PURE__ */ __name((list) => list.filter((flag) => !advertised.has(flag)), "missing");
|
|
8204
|
-
return {
|
|
8205
|
-
...base,
|
|
8206
|
-
supportedFlags: advertised,
|
|
8207
|
-
missingRequiredFlags: missing(required),
|
|
8208
|
-
missingConditionalFlags: missing(MODEL_CLI_CONDITIONAL_FLAGS[provider]),
|
|
8209
|
-
unsupportedOptionalFlags: missing(MODEL_CLI_OPTIONAL_FLAGS[provider])
|
|
8210
|
-
};
|
|
8211
|
-
}
|
|
8212
|
-
__name(judgeModelCliHelp, "judgeModelCliHelp");
|
|
8213
|
-
function isModelCliCompatible(compat) {
|
|
8214
|
-
return compat.missingRequiredFlags.length === 0;
|
|
8215
|
-
}
|
|
8216
|
-
__name(isModelCliCompatible, "isModelCliCompatible");
|
|
8217
|
-
function describeCliVersions(results) {
|
|
8218
|
-
return results.map((result) => `${result.provider} ${result.version ?? "unknown"}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH)).slice(0, FLEET_MAX_CLI_VERSIONS);
|
|
8219
|
-
}
|
|
8220
|
-
__name(describeCliVersions, "describeCliVersions");
|
|
8221
|
-
var LEGACY_COMMAND_NAME = {
|
|
8222
|
-
"claude-code": "claude",
|
|
8223
|
-
codex: "codex"
|
|
8224
|
-
};
|
|
8225
|
-
function primaryPinnedCliVersion(results) {
|
|
8226
|
-
const first = results.find((result) => result.version !== null);
|
|
8227
|
-
return first ? `${LEGACY_COMMAND_NAME[first.provider]} ${first.version}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH) : null;
|
|
8228
|
-
}
|
|
8229
|
-
__name(primaryPinnedCliVersion, "primaryPinnedCliVersion");
|
|
8230
|
-
|
|
8231
|
-
// src/core/worker-loop.ts
|
|
8232
|
-
var import_node_perf_hooks = require("node:perf_hooks");
|
|
8233
|
-
|
|
8234
|
-
// src/core/heartbeat.ts
|
|
8235
|
-
var systemScheduler = {
|
|
8236
|
-
setTimeout: /* @__PURE__ */ __name((callback, ms) => setTimeout(callback, ms), "setTimeout"),
|
|
8237
|
-
clearTimeout: /* @__PURE__ */ __name((handle) => clearTimeout(handle), "clearTimeout")
|
|
8238
|
-
};
|
|
8239
|
-
var OPTIONAL_DESCRIPTION_FIELD_TIERS = [
|
|
8240
|
-
// NEWEST first. Node lifecycle + remote node limits (self-build slices
|
|
8241
|
-
// AR, AS). Tiered (review): `cliVersions` goes out on EVERY beat of a
|
|
8242
|
-
// node with a pinned CLI, so an upgraded node talking to a platform that
|
|
8243
|
-
// predates it would otherwise drop the worker state and housekeeping
|
|
8244
|
-
// along with it — losing health signals that platform DOES understand,
|
|
8245
|
-
// for the life of the process. Stripping the newest tier first keeps them.
|
|
8246
|
-
["cliVersions", "maxConcurrentJobs", "maxCpuPercent", "maxMemoryMb"],
|
|
8247
|
-
// Fleet health signals (EW-776) and node housekeeping (EW-803).
|
|
8248
|
-
[
|
|
8249
|
-
"workerState",
|
|
8250
|
-
"workerStateReason",
|
|
8251
|
-
"minFreeDiskBytes",
|
|
8252
|
-
"workspaceCount",
|
|
8253
|
-
"workspaceBytes",
|
|
8254
|
-
"lastReclaimAt",
|
|
8255
|
-
"lastReclaimFreedBytes"
|
|
8256
|
-
]
|
|
8257
|
-
];
|
|
8258
|
-
function computeBackoffDelay(intervalMs, failures, maxMs) {
|
|
8259
|
-
if (failures <= 0) {
|
|
8260
|
-
return intervalMs;
|
|
8261
|
-
}
|
|
8262
|
-
const exponent = Math.min(failures, 30);
|
|
8263
|
-
const delay2 = intervalMs * 2 ** exponent;
|
|
8264
|
-
return Math.min(delay2, maxMs);
|
|
8265
|
-
}
|
|
8266
|
-
__name(computeBackoffDelay, "computeBackoffDelay");
|
|
8267
|
-
var HeartbeatLoop = class {
|
|
8268
|
-
static {
|
|
8269
|
-
__name(this, "HeartbeatLoop");
|
|
8270
|
-
}
|
|
8271
|
-
options;
|
|
8272
|
-
scheduler;
|
|
8273
|
-
now;
|
|
8274
|
-
listeners = /* @__PURE__ */ new Set();
|
|
8275
|
-
timer = null;
|
|
8276
|
-
running = false;
|
|
8277
|
-
inFlight = null;
|
|
8278
|
-
/**
|
|
8279
|
-
* How many optional-field TIERS (newest first) this platform has proven
|
|
8280
|
-
* it predates; from then on the beat leaves those tiers out. 0 = it
|
|
8281
|
-
* takes everything. Process-scoped on purpose — a platform upgrade is a
|
|
8282
|
-
* restart-shaped event for the node, and re-probing on every beat would
|
|
8283
|
-
* mean one wasted round trip per beat, forever, against an API that
|
|
8284
|
-
* will never accept them.
|
|
8285
|
-
*/
|
|
8286
|
-
strippedTiers = 0;
|
|
8287
|
-
state = {
|
|
8288
|
-
state: "idle",
|
|
8289
|
-
lastHeartbeatAt: null,
|
|
8290
|
-
consecutiveFailures: 0,
|
|
8291
|
-
nextAttemptInMs: null,
|
|
8292
|
-
lastError: null,
|
|
8293
|
-
lastErrorKind: null,
|
|
8294
|
-
node: null
|
|
8295
|
-
};
|
|
8296
|
-
constructor(options) {
|
|
8297
|
-
this.options = {
|
|
8298
|
-
...options,
|
|
8299
|
-
intervalMs: options.intervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS,
|
|
8300
|
-
maxBackoffMs: options.maxBackoffMs ?? MAX_HEARTBEAT_BACKOFF_MS
|
|
8301
|
-
};
|
|
8302
|
-
this.scheduler = options.scheduler ?? systemScheduler;
|
|
8303
|
-
this.now = options.now ?? (() => Date.now());
|
|
8304
|
-
this.options.logger?.protect(options.secret);
|
|
8305
|
-
}
|
|
8306
|
-
getState() {
|
|
8307
|
-
return { ...this.state };
|
|
8308
|
-
}
|
|
8309
|
-
onChange(listener) {
|
|
8310
|
-
this.listeners.add(listener);
|
|
8311
|
-
return () => {
|
|
8312
|
-
this.listeners.delete(listener);
|
|
8313
|
-
};
|
|
8314
|
-
}
|
|
8315
|
-
/** Start beating. Resolves once the FIRST beat has settled. */
|
|
8316
|
-
start() {
|
|
8317
|
-
if (this.running) {
|
|
8318
|
-
return this.inFlight ?? Promise.resolve();
|
|
8319
|
-
}
|
|
8320
|
-
this.running = true;
|
|
8321
|
-
return this.tick();
|
|
8322
|
-
}
|
|
8323
|
-
/** Stop beating and cancel any pending timer. */
|
|
8324
|
-
stop() {
|
|
8325
|
-
this.running = false;
|
|
8326
|
-
this.cancelTimer();
|
|
8327
|
-
this.patch({ state: "stopped", nextAttemptInMs: null });
|
|
8328
|
-
}
|
|
8329
|
-
/** Await the in-flight beat, if any (test/shutdown helper). */
|
|
8330
|
-
async settled() {
|
|
8331
|
-
await this.inFlight;
|
|
8332
|
-
}
|
|
8333
|
-
/** Run one beat now. Exposed so the UI can offer "retry" and tests can step. */
|
|
8334
|
-
tick() {
|
|
8335
|
-
const run = this.runOnce().finally(() => {
|
|
8336
|
-
if (this.inFlight === run) {
|
|
8337
|
-
this.inFlight = null;
|
|
8338
|
-
}
|
|
8339
|
-
});
|
|
8340
|
-
this.inFlight = run;
|
|
8341
|
-
return run;
|
|
8342
|
-
}
|
|
8343
|
-
async runOnce() {
|
|
8344
|
-
if (!this.running) {
|
|
8345
|
-
return;
|
|
8346
|
-
}
|
|
8347
|
-
this.cancelTimer();
|
|
8348
|
-
if (this.state.state !== "connected") {
|
|
8349
|
-
this.patch({ state: "connecting" });
|
|
8350
|
-
}
|
|
8351
|
-
try {
|
|
8352
|
-
const description = await this.options.describe();
|
|
8353
|
-
this.onSuccess(await this.beat(description));
|
|
8354
|
-
} catch (error) {
|
|
8355
|
-
this.onFailure(error);
|
|
8356
|
-
}
|
|
8357
|
-
this.scheduleNext();
|
|
8358
|
-
}
|
|
8359
|
-
/**
|
|
8360
|
-
* Send one beat, tolerating a platform that predates the optional
|
|
8361
|
-
* self-description fields (the worker state, EW-776; the housekeeping
|
|
8362
|
-
* report, EW-803).
|
|
8363
|
-
*
|
|
8364
|
-
* The API validates heartbeats with `whitelist + forbidNonWhitelisted`,
|
|
8365
|
-
* so a field an older build does not know is not ignored — it 400s the
|
|
8366
|
-
* whole request. A 400 is a FAILED beat, a failed beat backs off, and
|
|
8367
|
-
* enough of them sweep the node to `offline`. Which would mean shipping
|
|
8368
|
-
* a health-reporting feature whose failure mode is "every node in a
|
|
8369
|
-
* mixed-version fleet goes dark": the exact class of outage this slice
|
|
8370
|
-
* was written to end.
|
|
8371
|
-
*
|
|
8372
|
-
* So a 400 on a beat that CARRIED those fields is retried once,
|
|
8373
|
-
* immediately, without them. If that succeeds the platform is simply
|
|
8374
|
-
* older than this daemon; we latch, say so once, and keep reporting
|
|
8375
|
-
* liveness. The retry costs one request, once per process.
|
|
8376
|
-
*
|
|
8377
|
-
* Inferring "the field was rejected" from a bare 400 is deliberate and
|
|
8378
|
-
* has a cost worth stating: `FleetClient` never surfaces server bodies
|
|
8379
|
-
* (fixed client-authored messages only), so a 400 caused by something
|
|
8380
|
-
* else on such a beat also latches, and this node reports liveness only
|
|
8381
|
-
* until it restarts. That is the safe direction — the node stays
|
|
8382
|
-
* observable either way, and only the extra signal is lost.
|
|
8383
|
-
*/
|
|
8384
|
-
async beat(description) {
|
|
8385
|
-
const credential = { nodeId: this.options.nodeId, secret: this.options.secret };
|
|
8386
|
-
let sent = stripTiers(description, this.strippedTiers);
|
|
8387
|
-
try {
|
|
8388
|
-
return await this.options.client.heartbeat({ ...credential, ...sent });
|
|
8389
|
-
} catch (error) {
|
|
8390
|
-
if (!isFieldRejection(error)) throw error;
|
|
8391
|
-
let lastError = error;
|
|
8392
|
-
for (let tiers = this.strippedTiers + 1; tiers <= OPTIONAL_DESCRIPTION_FIELD_TIERS.length; tiers += 1) {
|
|
8393
|
-
const candidate = stripTiers(description, tiers);
|
|
8394
|
-
if (Object.keys(candidate).length === Object.keys(sent).length) continue;
|
|
8395
|
-
sent = candidate;
|
|
8396
|
-
try {
|
|
8397
|
-
const result = await this.options.client.heartbeat({ ...credential, ...candidate });
|
|
8398
|
-
this.strippedTiers = tiers;
|
|
8399
|
-
this.options.logger?.warn(
|
|
8400
|
-
tiers >= OPTIONAL_DESCRIPTION_FIELD_TIERS.length ? "Platform rejected the optional self-description fields (worker state, housekeeping); it predates them. Reporting liveness only until this node restarts." : "Platform rejected the newest optional self-description fields (pinned CLI versions, enforced limits); it predates them. Still reporting worker state and housekeeping; the newer fields stay off until this node restarts."
|
|
8401
|
-
);
|
|
8402
|
-
return result;
|
|
8403
|
-
} catch (retryError) {
|
|
8404
|
-
if (!isFieldRejection(retryError)) throw retryError;
|
|
8405
|
-
lastError = retryError;
|
|
8406
|
-
}
|
|
8407
|
-
}
|
|
8408
|
-
throw lastError;
|
|
8409
|
-
}
|
|
8410
|
-
}
|
|
8411
|
-
onSuccess(result) {
|
|
8412
|
-
this.patch({
|
|
8413
|
-
state: "connected",
|
|
8414
|
-
lastHeartbeatAt: this.now(),
|
|
8415
|
-
consecutiveFailures: 0,
|
|
8416
|
-
lastError: null,
|
|
8417
|
-
lastErrorKind: null,
|
|
8418
|
-
node: result.node
|
|
8419
|
-
});
|
|
8420
|
-
if (this.options.onAccepted) {
|
|
8421
|
-
try {
|
|
8422
|
-
this.options.onAccepted(result);
|
|
8423
|
-
} catch {
|
|
8424
|
-
}
|
|
8425
|
-
}
|
|
8426
|
-
}
|
|
8427
|
-
onFailure(error) {
|
|
8428
|
-
const kind = error instanceof FleetClientError ? error.kind : "network";
|
|
8429
|
-
const raw = error instanceof Error ? error.message : String(error);
|
|
8430
|
-
const message = this.options.logger?.redact(raw) ?? raw;
|
|
8431
|
-
const failures = this.state.consecutiveFailures + 1;
|
|
8432
|
-
this.patch({
|
|
8433
|
-
// `unauthorized` is sticky and visible: the operator has to act
|
|
8434
|
-
// (re-enable the node in Fleet, or re-enroll). We keep retrying at
|
|
8435
|
-
// the backoff ceiling so a re-enabled node recovers on its own.
|
|
8436
|
-
state: kind === "unauthorized" ? "unauthorized" : "retrying",
|
|
8437
|
-
consecutiveFailures: failures,
|
|
8438
|
-
lastError: message,
|
|
8439
|
-
lastErrorKind: kind
|
|
8778
|
+
this.cache.set(`${provider}|${executable}`, {
|
|
8779
|
+
key,
|
|
8780
|
+
// A COMPLETE answer from a stamped file is kept until the file
|
|
8781
|
+
// changes or, at the latest, for `stampedMaxAgeMs` (an upgrade
|
|
8782
|
+
// behind a stable wrapper does not change the stamp). An
|
|
8783
|
+
// incomplete one is retried soon, and an unstamped one on the
|
|
8784
|
+
// ten-minute clock.
|
|
8785
|
+
ttlMs: !answered ? this.incompleteRetryMs : stamp !== null ? this.stampedMaxAgeMs : this.unstampedTtlMs,
|
|
8786
|
+
probedAt: this.now(),
|
|
8787
|
+
result
|
|
8788
|
+
});
|
|
8789
|
+
return result;
|
|
8790
|
+
}).finally(() => {
|
|
8791
|
+
this.inFlight.delete(key);
|
|
8440
8792
|
});
|
|
8441
|
-
this.
|
|
8793
|
+
this.inFlight.set(key, run);
|
|
8794
|
+
return run;
|
|
8442
8795
|
}
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8796
|
+
/** Every pinned binary in `paths`, in provider order. Unpinned providers are skipped. */
|
|
8797
|
+
async probeAll(paths) {
|
|
8798
|
+
const out = [];
|
|
8799
|
+
for (const provider of MODEL_CLI_PROVIDERS) {
|
|
8800
|
+
const executable = paths?.[provider];
|
|
8801
|
+
if (typeof executable !== "string" || !executable.trim()) continue;
|
|
8802
|
+
out.push(await this.probe(provider, executable));
|
|
8446
8803
|
}
|
|
8447
|
-
|
|
8448
|
-
this.options.intervalMs,
|
|
8449
|
-
this.state.consecutiveFailures,
|
|
8450
|
-
this.options.maxBackoffMs
|
|
8451
|
-
);
|
|
8452
|
-
this.patch({ nextAttemptInMs: delay2 });
|
|
8453
|
-
this.timer = this.scheduler.setTimeout(() => {
|
|
8454
|
-
this.timer = null;
|
|
8455
|
-
void this.tick();
|
|
8456
|
-
}, delay2);
|
|
8804
|
+
return out;
|
|
8457
8805
|
}
|
|
8458
|
-
|
|
8459
|
-
if (this.
|
|
8460
|
-
|
|
8461
|
-
|
|
8806
|
+
async stamp(executable) {
|
|
8807
|
+
if (!this.options.statFile) return null;
|
|
8808
|
+
try {
|
|
8809
|
+
const stamp = await this.options.statFile(executable);
|
|
8810
|
+
return stamp && Number.isFinite(stamp.mtimeMs) && Number.isFinite(stamp.size) ? stamp : null;
|
|
8811
|
+
} catch {
|
|
8812
|
+
return null;
|
|
8462
8813
|
}
|
|
8463
8814
|
}
|
|
8464
|
-
|
|
8465
|
-
|
|
8466
|
-
const
|
|
8467
|
-
|
|
8468
|
-
|
|
8815
|
+
async run(provider, executable) {
|
|
8816
|
+
const command = this.commandFor(executable);
|
|
8817
|
+
const versionOutput = await this.ask(command, ["--version"]);
|
|
8818
|
+
const version = versionOutput === null ? null : parseCliVersion(versionOutput);
|
|
8819
|
+
const help = await this.ask(command, [...HELP_ARGS[provider]]);
|
|
8820
|
+
return {
|
|
8821
|
+
result: judgeModelCliHelp(provider, executable, version, help),
|
|
8822
|
+
// "Answered" = both invocations produced output. An UNRECOGNISED
|
|
8823
|
+
// help text is still an answer — deterministic for that file — and
|
|
8824
|
+
// re-asking every minute would learn nothing.
|
|
8825
|
+
answered: versionOutput !== null && help !== null
|
|
8826
|
+
};
|
|
8827
|
+
}
|
|
8828
|
+
/**
|
|
8829
|
+
* The runner on Windows goes through `cmd.exe` (`shell: true`), which
|
|
8830
|
+
* splits an unquoted `C:\Program Files\…` at the space.
|
|
8831
|
+
*/
|
|
8832
|
+
commandFor(executable) {
|
|
8833
|
+
return this.options.platform === "win32" && /\s/.test(executable) && !executable.startsWith('"') ? `"${executable}"` : executable;
|
|
8834
|
+
}
|
|
8835
|
+
/** stdout+stderr of one invocation, or null when it could not be run at all. */
|
|
8836
|
+
async ask(command, args) {
|
|
8837
|
+
try {
|
|
8838
|
+
const result = await this.options.runner.run(command, args);
|
|
8839
|
+
if (result.timedOut) return null;
|
|
8840
|
+
const text = `${result.stdout ?? ""}
|
|
8841
|
+
${result.stderr ?? ""}`;
|
|
8842
|
+
return result.code === 0 || text.trim() ? text : null;
|
|
8843
|
+
} catch {
|
|
8844
|
+
return null;
|
|
8469
8845
|
}
|
|
8470
8846
|
}
|
|
8471
8847
|
};
|
|
8472
|
-
function
|
|
8473
|
-
const
|
|
8474
|
-
const
|
|
8475
|
-
for (const
|
|
8476
|
-
|
|
8477
|
-
out[key] = value;
|
|
8848
|
+
function parseAdvertisedFlags(help) {
|
|
8849
|
+
const flags = /* @__PURE__ */ new Set();
|
|
8850
|
+
const pattern = /(?:^|[\s,[(|])(--?[A-Za-z][A-Za-z0-9-]*)(?=$|[\s,=<[\])|.:])/gm;
|
|
8851
|
+
for (const match of help.matchAll(pattern)) {
|
|
8852
|
+
flags.add(match[1]);
|
|
8478
8853
|
}
|
|
8479
|
-
return
|
|
8854
|
+
return flags;
|
|
8480
8855
|
}
|
|
8481
|
-
__name(
|
|
8482
|
-
function
|
|
8483
|
-
|
|
8856
|
+
__name(parseAdvertisedFlags, "parseAdvertisedFlags");
|
|
8857
|
+
function judgeModelCliHelp(provider, executable, version, help) {
|
|
8858
|
+
const required = MODEL_CLI_REQUIRED_FLAGS[provider];
|
|
8859
|
+
const base = {
|
|
8860
|
+
provider,
|
|
8861
|
+
executable,
|
|
8862
|
+
version,
|
|
8863
|
+
supportedFlags: null,
|
|
8864
|
+
missingRequiredFlags: [],
|
|
8865
|
+
missingConditionalFlags: [],
|
|
8866
|
+
unsupportedOptionalFlags: [],
|
|
8867
|
+
note: null
|
|
8868
|
+
};
|
|
8869
|
+
if (help === null) {
|
|
8870
|
+
return { ...base, note: "the binary did not answer --help; nothing is dropped" };
|
|
8871
|
+
}
|
|
8872
|
+
const advertised = parseAdvertisedFlags(help);
|
|
8873
|
+
if (!required.some((flag) => advertised.has(flag))) {
|
|
8874
|
+
return {
|
|
8875
|
+
...base,
|
|
8876
|
+
note: `the help text lists none of the flags this node always passes (${required.join(", ")}); its format is not recognised, so nothing is dropped`
|
|
8877
|
+
};
|
|
8878
|
+
}
|
|
8879
|
+
const missing = /* @__PURE__ */ __name((list) => list.filter((flag) => !advertised.has(flag)), "missing");
|
|
8880
|
+
return {
|
|
8881
|
+
...base,
|
|
8882
|
+
supportedFlags: advertised,
|
|
8883
|
+
missingRequiredFlags: missing(required),
|
|
8884
|
+
missingConditionalFlags: missing(MODEL_CLI_CONDITIONAL_FLAGS[provider]),
|
|
8885
|
+
unsupportedOptionalFlags: missing(MODEL_CLI_OPTIONAL_FLAGS[provider])
|
|
8886
|
+
};
|
|
8484
8887
|
}
|
|
8485
|
-
__name(
|
|
8888
|
+
__name(judgeModelCliHelp, "judgeModelCliHelp");
|
|
8889
|
+
function isModelCliCompatible(compat) {
|
|
8890
|
+
return compat.missingRequiredFlags.length === 0;
|
|
8891
|
+
}
|
|
8892
|
+
__name(isModelCliCompatible, "isModelCliCompatible");
|
|
8893
|
+
function describeCliVersions(results) {
|
|
8894
|
+
return results.map((result) => `${result.provider} ${result.version ?? "unknown"}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH)).slice(0, FLEET_MAX_CLI_VERSIONS);
|
|
8895
|
+
}
|
|
8896
|
+
__name(describeCliVersions, "describeCliVersions");
|
|
8897
|
+
var LEGACY_COMMAND_NAME = {
|
|
8898
|
+
"claude-code": "claude",
|
|
8899
|
+
codex: "codex"
|
|
8900
|
+
};
|
|
8901
|
+
function primaryPinnedCliVersion(results) {
|
|
8902
|
+
const first = results.find((result) => result.version !== null);
|
|
8903
|
+
return first ? `${LEGACY_COMMAND_NAME[first.provider]} ${first.version}`.slice(0, FLEET_MAX_CLI_VERSION_LENGTH) : null;
|
|
8904
|
+
}
|
|
8905
|
+
__name(primaryPinnedCliVersion, "primaryPinnedCliVersion");
|
|
8906
|
+
|
|
8907
|
+
// src/core/worker-loop.ts
|
|
8908
|
+
var import_node_perf_hooks = require("node:perf_hooks");
|
|
8486
8909
|
|
|
8487
8910
|
// src/core/resource-limits.ts
|
|
8488
8911
|
var ADMIT = { admit: true, reason: null };
|
|
@@ -8577,6 +9000,7 @@ __name(measureWorkspaceFreeBytes, "measureWorkspaceFreeBytes");
|
|
|
8577
9000
|
|
|
8578
9001
|
// src/core/worker-loop.ts
|
|
8579
9002
|
var DEFAULT_IDLE_POLL_MS = 5e3;
|
|
9003
|
+
var MODEL_THROTTLE_SYNC_GRACE_MS = 12e4;
|
|
8580
9004
|
var WORKER_BACKOFF_BASE_MS = 1e3;
|
|
8581
9005
|
var WORKER_BACKOFF_MAX_MS = 6e4;
|
|
8582
9006
|
var MIN_KEEPALIVE_MS = 5e3;
|
|
@@ -8675,6 +9099,8 @@ var WorkerLoop = class {
|
|
|
8675
9099
|
monotonicNow;
|
|
8676
9100
|
leasePollDrainTimeoutMs;
|
|
8677
9101
|
publishFenceMarginMs;
|
|
9102
|
+
/** Fleet usage limits: provider → the throttle in force (expiry checked on read). */
|
|
9103
|
+
modelThrottles = /* @__PURE__ */ new Map();
|
|
8678
9104
|
running = false;
|
|
8679
9105
|
stopping = false;
|
|
8680
9106
|
paused = false;
|
|
@@ -8734,7 +9160,118 @@ var WorkerLoop = class {
|
|
|
8734
9160
|
return [...this.executors.keys()];
|
|
8735
9161
|
}
|
|
8736
9162
|
getState() {
|
|
8737
|
-
|
|
9163
|
+
const modelThrottles = this.activeModelThrottles();
|
|
9164
|
+
return {
|
|
9165
|
+
...this.state,
|
|
9166
|
+
activeJobIds: [...this.state.activeJobIds],
|
|
9167
|
+
...modelThrottles.length > 0 ? { modelThrottles } : {}
|
|
9168
|
+
};
|
|
9169
|
+
}
|
|
9170
|
+
/**
|
|
9171
|
+
* Fleet usage limits — record that `limit.provider` is out of quota on this
|
|
9172
|
+
* machine until `limit.retryAfter`. From the next poll the loop leases no
|
|
9173
|
+
* `agent-task` for that CLI (all of `agent-task` when every CLI this node
|
|
9174
|
+
* has is out), and refuses — without starting it — any such job the
|
|
9175
|
+
* platform still hands it. Other job kinds keep leasing. A later reset
|
|
9176
|
+
* replaces an earlier one; an earlier one never shortens a later one.
|
|
9177
|
+
*/
|
|
9178
|
+
noteModelUsageLimit(limit) {
|
|
9179
|
+
const provider = typeof limit.provider === "string" ? limit.provider.trim() : "";
|
|
9180
|
+
const until = Date.parse(limit.retryAfter);
|
|
9181
|
+
if (!provider || !Number.isFinite(until) || until <= this.now()) return;
|
|
9182
|
+
const previous = this.modelThrottles.get(provider);
|
|
9183
|
+
if (previous && previous.until >= until) return;
|
|
9184
|
+
this.modelThrottles.set(provider, { until, window: limit.window, reason: limit.message, setAt: this.now() });
|
|
9185
|
+
if (!previous || previous.until <= this.now()) {
|
|
9186
|
+
this.options.logger?.warn(
|
|
9187
|
+
`${describeFleetModelProvider(provider)} hit its ${describeFleetModelUsageLimitWindow(limit.window)} on this node \u2014 not leasing ${provider} agent-task jobs until ${new Date(until).toISOString()} (other work still runs)`
|
|
9188
|
+
);
|
|
9189
|
+
}
|
|
9190
|
+
this.patch({});
|
|
9191
|
+
}
|
|
9192
|
+
/**
|
|
9193
|
+
* Fleet usage limits — reconcile with the platform's view of this node's
|
|
9194
|
+
* throttles, as carried on every heartbeat response (`node.modelThrottles`).
|
|
9195
|
+
*
|
|
9196
|
+
* The platform is authoritative both ways:
|
|
9197
|
+
* - a throttle it holds that this loop does not (a restarted daemon, a
|
|
9198
|
+
* limit another path recorded) is ADOPTED, so the node stops offering
|
|
9199
|
+
* itself for that CLI and says so on its next beat;
|
|
9200
|
+
* - a throttle this loop holds that the platform no longer does — the
|
|
9201
|
+
* owner pressed "clear limit now" — is LIFTED, but only once it is older
|
|
9202
|
+
* than `graceMs`: a heartbeat answered just before the platform recorded
|
|
9203
|
+
* the limit this node just reported must not undo it.
|
|
9204
|
+
* `null` / absent (an API older than the field) changes nothing.
|
|
9205
|
+
*/
|
|
9206
|
+
syncModelThrottles(server, graceMs = MODEL_THROTTLE_SYNC_GRACE_MS) {
|
|
9207
|
+
if (!Array.isArray(server)) return;
|
|
9208
|
+
const now = this.now();
|
|
9209
|
+
const held = /* @__PURE__ */ new Set();
|
|
9210
|
+
let changed = false;
|
|
9211
|
+
for (const entry of server) {
|
|
9212
|
+
const provider = typeof entry?.provider === "string" ? entry.provider.trim() : "";
|
|
9213
|
+
const until = typeof entry?.until === "string" ? Date.parse(entry.until) : Number.NaN;
|
|
9214
|
+
if (!provider || !Number.isFinite(until) || until <= now) continue;
|
|
9215
|
+
held.add(provider);
|
|
9216
|
+
const local = this.modelThrottles.get(provider);
|
|
9217
|
+
if (local && local.until >= until) continue;
|
|
9218
|
+
const window = typeof entry.window === "string" ? entry.window : "usage";
|
|
9219
|
+
const reason = typeof entry.reason === "string" ? entry.reason : `${provider} usage limit reached`;
|
|
9220
|
+
this.modelThrottles.set(provider, { until, window, reason, setAt: local?.setAt ?? now });
|
|
9221
|
+
changed = true;
|
|
9222
|
+
if (!local) {
|
|
9223
|
+
this.options.logger?.info(
|
|
9224
|
+
`${describeFleetModelProvider(provider)} is out of quota on this node per the platform \u2014 not leasing ${provider} agent-task jobs until ${new Date(until).toISOString()}`
|
|
9225
|
+
);
|
|
9226
|
+
}
|
|
9227
|
+
}
|
|
9228
|
+
for (const [provider, local] of this.modelThrottles) {
|
|
9229
|
+
if (held.has(provider) || now - local.setAt < graceMs) continue;
|
|
9230
|
+
this.modelThrottles.delete(provider);
|
|
9231
|
+
changed = true;
|
|
9232
|
+
this.options.logger?.info(
|
|
9233
|
+
`${describeFleetModelProvider(provider)} usage limit was cleared on the platform \u2014 leasing ${provider} agent-task jobs again`
|
|
9234
|
+
);
|
|
9235
|
+
}
|
|
9236
|
+
if (changed) {
|
|
9237
|
+
this.patch({});
|
|
9238
|
+
this.pollSoon();
|
|
9239
|
+
}
|
|
9240
|
+
}
|
|
9241
|
+
/** The usage-limit throttles still in force; expired ones are dropped (and logged) here. */
|
|
9242
|
+
activeModelThrottles() {
|
|
9243
|
+
const now = this.now();
|
|
9244
|
+
const active = [];
|
|
9245
|
+
for (const [provider, throttle] of this.modelThrottles) {
|
|
9246
|
+
if (throttle.until <= now) {
|
|
9247
|
+
this.modelThrottles.delete(provider);
|
|
9248
|
+
this.options.logger?.info(
|
|
9249
|
+
`${describeFleetModelProvider(provider)} usage limit has reset \u2014 leasing ${provider} agent-task jobs again`
|
|
9250
|
+
);
|
|
9251
|
+
continue;
|
|
9252
|
+
}
|
|
9253
|
+
active.push({
|
|
9254
|
+
provider,
|
|
9255
|
+
window: throttle.window,
|
|
9256
|
+
until: new Date(throttle.until).toISOString(),
|
|
9257
|
+
reason: throttle.reason
|
|
9258
|
+
});
|
|
9259
|
+
}
|
|
9260
|
+
return active.sort((a, b) => a.provider.localeCompare(b.provider));
|
|
9261
|
+
}
|
|
9262
|
+
/** True when every model CLI this node can run an agent-task with is out of quota. */
|
|
9263
|
+
allAgentCliThrottled() {
|
|
9264
|
+
const active = this.activeModelThrottles();
|
|
9265
|
+
if (active.length === 0) return false;
|
|
9266
|
+
let providers = [];
|
|
9267
|
+
try {
|
|
9268
|
+
providers = this.options.modelProviders?.() ?? [];
|
|
9269
|
+
} catch {
|
|
9270
|
+
providers = [];
|
|
9271
|
+
}
|
|
9272
|
+
if (providers.length === 0) return true;
|
|
9273
|
+
const out = new Set(active.map((throttle) => throttle.provider));
|
|
9274
|
+
return providers.every((provider) => out.has(provider));
|
|
8738
9275
|
}
|
|
8739
9276
|
onChange(listener) {
|
|
8740
9277
|
this.listeners.add(listener);
|
|
@@ -9012,8 +9549,10 @@ var WorkerLoop = class {
|
|
|
9012
9549
|
};
|
|
9013
9550
|
if (this.options.capabilities) request.capabilities = this.options.capabilities;
|
|
9014
9551
|
if (this.options.kinds && this.options.kinds.length > 0) request.kinds = [...this.options.kinds];
|
|
9015
|
-
|
|
9016
|
-
|
|
9552
|
+
const excludeKinds = new Set(this.options.excludeKinds ?? []);
|
|
9553
|
+
if (this.allAgentCliThrottled()) excludeKinds.add("agent-task");
|
|
9554
|
+
if (excludeKinds.size > 0) {
|
|
9555
|
+
request.excludeKinds = [...excludeKinds];
|
|
9017
9556
|
}
|
|
9018
9557
|
if (this.options.client.leaseOutcome) {
|
|
9019
9558
|
const outcome = await this.options.client.leaseOutcome(request, pollController.signal);
|
|
@@ -9250,6 +9789,14 @@ var WorkerLoop = class {
|
|
|
9250
9789
|
);
|
|
9251
9790
|
return;
|
|
9252
9791
|
}
|
|
9792
|
+
const throttled = this.throttledResultFor(job);
|
|
9793
|
+
if (throttled) {
|
|
9794
|
+
this.options.logger?.info(
|
|
9795
|
+
`Fleet job ${job.id} not started: ${String(throttled.failureReason)} \u2014 handed back for the platform to park`
|
|
9796
|
+
);
|
|
9797
|
+
await this.report(job.id, { success: true, result: throttled }, generation, run);
|
|
9798
|
+
return;
|
|
9799
|
+
}
|
|
9253
9800
|
const keepAlive = this.startKeepAlive(run);
|
|
9254
9801
|
let successAccepted = false;
|
|
9255
9802
|
try {
|
|
@@ -9258,6 +9805,7 @@ var WorkerLoop = class {
|
|
|
9258
9805
|
);
|
|
9259
9806
|
const result = await executor(job, signal, keepAlive.handle);
|
|
9260
9807
|
throwIfJobAborted(signal);
|
|
9808
|
+
this.noteUsageLimitResult(result);
|
|
9261
9809
|
if (this.settleDeferred(job.id, keepAlive.deferral())) return;
|
|
9262
9810
|
const accepted = await this.report(
|
|
9263
9811
|
job.id,
|
|
@@ -9292,6 +9840,57 @@ var WorkerLoop = class {
|
|
|
9292
9840
|
keepAlive.stop();
|
|
9293
9841
|
}
|
|
9294
9842
|
}
|
|
9843
|
+
/** A result that says its CLI hit a usage limit throttles that CLI here. */
|
|
9844
|
+
noteUsageLimitResult(result) {
|
|
9845
|
+
if (!result || result.failureKind !== FLEET_JOB_FAILURE_KIND_MODEL_USAGE_LIMIT) return;
|
|
9846
|
+
const limit = result.modelUsageLimit;
|
|
9847
|
+
if (!limit || typeof limit.provider !== "string" || typeof limit.retryAfter !== "string") return;
|
|
9848
|
+
this.noteModelUsageLimit({
|
|
9849
|
+
provider: limit.provider,
|
|
9850
|
+
window: limit.window ?? "usage",
|
|
9851
|
+
retryAfter: limit.retryAfter,
|
|
9852
|
+
message: typeof limit.message === "string" ? limit.message : `${limit.provider} usage limit reached`
|
|
9853
|
+
});
|
|
9854
|
+
}
|
|
9855
|
+
/**
|
|
9856
|
+
* The usage-limit outcome for an `agent-task` whose CLI is out of quota on
|
|
9857
|
+
* this node, or null. Shaped like a real run's result (model block
|
|
9858
|
+
* included) so the platform — current or older — reads it the same way.
|
|
9859
|
+
*/
|
|
9860
|
+
throttledResultFor(job) {
|
|
9861
|
+
if (job.kind !== "agent-task") return null;
|
|
9862
|
+
const payload = job.payload ?? {};
|
|
9863
|
+
const execution = payload.execution;
|
|
9864
|
+
const provider = typeof execution?.provider === "string" ? execution.provider : null;
|
|
9865
|
+
if (!provider) return null;
|
|
9866
|
+
const throttle = this.activeModelThrottles().find((entry) => entry.provider === provider);
|
|
9867
|
+
if (!throttle) return null;
|
|
9868
|
+
const failureReason = `${provider} is out of quota on this node until ${throttle.until} (${describeFleetModelUsageLimitWindow(throttle.window)}) \u2014 not started`;
|
|
9869
|
+
return {
|
|
9870
|
+
status: "failed",
|
|
9871
|
+
taskId: typeof payload.taskId === "string" ? payload.taskId : "",
|
|
9872
|
+
runId: typeof payload.runId === "string" ? payload.runId : null,
|
|
9873
|
+
workspace: null,
|
|
9874
|
+
steps: [],
|
|
9875
|
+
model: {
|
|
9876
|
+
provider,
|
|
9877
|
+
status: "failed",
|
|
9878
|
+
exitCode: null,
|
|
9879
|
+
durationMs: 0,
|
|
9880
|
+
summary: throttle.reason
|
|
9881
|
+
},
|
|
9882
|
+
gateStatus: "none",
|
|
9883
|
+
failureReason,
|
|
9884
|
+
failureKind: FLEET_JOB_FAILURE_KIND_MODEL_USAGE_LIMIT,
|
|
9885
|
+
modelUsageLimit: {
|
|
9886
|
+
provider,
|
|
9887
|
+
window: throttle.window,
|
|
9888
|
+
resetsAt: null,
|
|
9889
|
+
retryAfter: throttle.until,
|
|
9890
|
+
message: throttle.reason
|
|
9891
|
+
}
|
|
9892
|
+
};
|
|
9893
|
+
}
|
|
9295
9894
|
/**
|
|
9296
9895
|
* Record a job the executor handed back UNSETTLED, and report nothing.
|
|
9297
9896
|
*
|
|
@@ -10882,6 +11481,32 @@ __name(isReportableBytes, "isReportableBytes");
|
|
|
10882
11481
|
// src/core/worker-health.ts
|
|
10883
11482
|
function describeWorkerHealth(state) {
|
|
10884
11483
|
if (!state) return null;
|
|
11484
|
+
const projected = projectWorkerState(state);
|
|
11485
|
+
const usage = describeModelThrottles(state);
|
|
11486
|
+
if (!usage) return projected;
|
|
11487
|
+
switch (projected.workerState) {
|
|
11488
|
+
case "idle":
|
|
11489
|
+
return health("throttled", usage);
|
|
11490
|
+
case "throttled":
|
|
11491
|
+
case "working":
|
|
11492
|
+
return health(
|
|
11493
|
+
projected.workerState,
|
|
11494
|
+
projected.workerStateReason ? `${projected.workerStateReason}; ${usage}` : usage
|
|
11495
|
+
);
|
|
11496
|
+
default:
|
|
11497
|
+
return projected;
|
|
11498
|
+
}
|
|
11499
|
+
}
|
|
11500
|
+
__name(describeWorkerHealth, "describeWorkerHealth");
|
|
11501
|
+
function describeModelThrottles(state) {
|
|
11502
|
+
const throttles = state.modelThrottles ?? [];
|
|
11503
|
+
if (throttles.length === 0) return null;
|
|
11504
|
+
return throttles.map(
|
|
11505
|
+
(throttle) => `${throttle.provider}: ${describeFleetModelUsageLimitWindow(throttle.window)} until ${throttle.until}`
|
|
11506
|
+
).join("; ");
|
|
11507
|
+
}
|
|
11508
|
+
__name(describeModelThrottles, "describeModelThrottles");
|
|
11509
|
+
function projectWorkerState(state) {
|
|
10885
11510
|
switch (state.state) {
|
|
10886
11511
|
// The fail-closed stop. `lastError` carries the quarantine's own
|
|
10887
11512
|
// message (`markUnsafe` writes it there), which is the single most
|
|
@@ -10911,7 +11536,7 @@ function describeWorkerHealth(state) {
|
|
|
10911
11536
|
return health("idle", null);
|
|
10912
11537
|
}
|
|
10913
11538
|
}
|
|
10914
|
-
__name(
|
|
11539
|
+
__name(projectWorkerState, "projectWorkerState");
|
|
10915
11540
|
function health(workerState, reason) {
|
|
10916
11541
|
const trimmed = typeof reason === "string" ? reason.trim() : "";
|
|
10917
11542
|
if (!trimmed) {
|
|
@@ -12871,6 +13496,31 @@ async function runResolvedAgentTask(context, io, signal) {
|
|
|
12871
13496
|
containment = modelStep.containment;
|
|
12872
13497
|
throwIfAgentTaskAborted(signal);
|
|
12873
13498
|
if (model.status !== "succeeded") {
|
|
13499
|
+
const usageLimit = recognizeModelUsageLimit(model, io);
|
|
13500
|
+
if (usageLimit) {
|
|
13501
|
+
if (workspaceResolution.descriptor && io.removeRunEnvFiles) {
|
|
13502
|
+
await io.removeRunEnvFiles(taskId, workspaceResolution.descriptor).catch(() => void 0);
|
|
13503
|
+
}
|
|
13504
|
+
io.logger?.warn(
|
|
13505
|
+
`[fleet-node] job ${job.id}: ${model.provider} hit a usage limit (${usageLimit.window}) \u2014 retry after ${usageLimit.retryAfter}; steps, checks and publish skipped`
|
|
13506
|
+
);
|
|
13507
|
+
return fitNodeOutcomeToResultBudget({
|
|
13508
|
+
status: "failed",
|
|
13509
|
+
taskId,
|
|
13510
|
+
runId,
|
|
13511
|
+
workspace: workspaceResolution.descriptor,
|
|
13512
|
+
steps: [],
|
|
13513
|
+
...setupResults.length > 0 ? { setup: setupResults } : {},
|
|
13514
|
+
...setup.length > 0 ? { setupStatus } : {},
|
|
13515
|
+
model,
|
|
13516
|
+
gateStatus: "none",
|
|
13517
|
+
...mcp ? { mcp } : {},
|
|
13518
|
+
...containment ? { containment } : {},
|
|
13519
|
+
failureReason: `${describeModelFailure(model)} \u2014 usage limit, retry after ${usageLimit.retryAfter}`,
|
|
13520
|
+
failureKind: FLEET_JOB_FAILURE_KIND_MODEL_USAGE_LIMIT,
|
|
13521
|
+
modelUsageLimit: usageLimit
|
|
13522
|
+
});
|
|
13523
|
+
}
|
|
12874
13524
|
failures.push(describeModelFailure(model));
|
|
12875
13525
|
}
|
|
12876
13526
|
}
|
|
@@ -13096,6 +13746,11 @@ async function runModelStep(jobId, execution, workspacePath, mounts, bridgeSpec,
|
|
|
13096
13746
|
emptyMcpConfigPath = mcpConfigPath;
|
|
13097
13747
|
}
|
|
13098
13748
|
const droppedFlags = await optionalFlagsToDrop(jobId, execution, executable, io);
|
|
13749
|
+
const codexUserSettings = execution.provider === "codex" ? await readOwnerCodexSettings(
|
|
13750
|
+
jobId,
|
|
13751
|
+
io,
|
|
13752
|
+
effectiveCodexHome(io, modelEnvOverlay, execution.envPassthrough, execution.envGrants)
|
|
13753
|
+
) : void 0;
|
|
13099
13754
|
const invoke = /* @__PURE__ */ __name(async (instructions, resumeSessionId) => {
|
|
13100
13755
|
await scratchFs.writeFile(scratch.instructionsPath, instructions);
|
|
13101
13756
|
let command;
|
|
@@ -13108,6 +13763,7 @@ async function runModelStep(jobId, execution, workspacePath, mounts, bridgeSpec,
|
|
|
13108
13763
|
...mounts && mounts.length > 0 ? { mounts } : {},
|
|
13109
13764
|
...bridge.cli ? { mcp: bridge.cli } : {},
|
|
13110
13765
|
...emptyMcpConfigPath ? { emptyMcpConfigPath } : {},
|
|
13766
|
+
...codexUserSettings ? { codexUserSettings } : {},
|
|
13111
13767
|
...resumeSessionId ? { resumeSessionId } : {},
|
|
13112
13768
|
...io.platform ? { platform: io.platform } : {},
|
|
13113
13769
|
...streamClaude ? { stream: true } : {},
|
|
@@ -13326,6 +13982,32 @@ function resolveRealHomeDir(io) {
|
|
|
13326
13982
|
return (0, import_os2.homedir)();
|
|
13327
13983
|
}
|
|
13328
13984
|
__name(resolveRealHomeDir, "resolveRealHomeDir");
|
|
13985
|
+
async function readOwnerCodexSettings(jobId, io, codexHome) {
|
|
13986
|
+
const read = io.sessionConfigFs?.readFile?.bind(io.sessionConfigFs);
|
|
13987
|
+
if (!read) return {};
|
|
13988
|
+
try {
|
|
13989
|
+
return readCodexUserSettings(await read((0, import_path4.join)(codexHome, "config.toml")));
|
|
13990
|
+
} catch (error) {
|
|
13991
|
+
io.logger?.warn(
|
|
13992
|
+
`[fleet-node] job ${jobId}: could not read the Codex config to keep its sandbox and model defaults; the run uses Codex's own defaults (still under --ignore-user-config): ${describeContainmentError(error)}`
|
|
13993
|
+
);
|
|
13994
|
+
return {};
|
|
13995
|
+
}
|
|
13996
|
+
}
|
|
13997
|
+
__name(readOwnerCodexSettings, "readOwnerCodexSettings");
|
|
13998
|
+
function effectiveCodexHome(io, envOverlay, envPassthrough, envGrants) {
|
|
13999
|
+
const isCodexHome = /* @__PURE__ */ __name((name) => name.toUpperCase() === "CODEX_HOME", "isCodexHome");
|
|
14000
|
+
const childEnv = buildNodeCheckEnv(
|
|
14001
|
+
envPassthrough?.filter(isCodexHome) ?? null,
|
|
14002
|
+
io.parentEnv,
|
|
14003
|
+
envGrants?.filter(isCodexHome) ?? null,
|
|
14004
|
+
envOverlay
|
|
14005
|
+
);
|
|
14006
|
+
const key = Object.keys(childEnv).find(isCodexHome);
|
|
14007
|
+
const value = key === void 0 ? void 0 : childEnv[key];
|
|
14008
|
+
return typeof value === "string" && value.trim() ? value.trim() : (0, import_path4.join)(resolveRealHomeDir(io), ".codex");
|
|
14009
|
+
}
|
|
14010
|
+
__name(effectiveCodexHome, "effectiveCodexHome");
|
|
13329
14011
|
function describeContainmentError(error) {
|
|
13330
14012
|
return error instanceof Error ? error.message : String(error);
|
|
13331
14013
|
}
|
|
@@ -13544,6 +14226,22 @@ function resumeNeverOpened(model) {
|
|
|
13544
14226
|
return RESUME_NEVER_OPENED_SIGNATURES.some((signature) => signature.test(said));
|
|
13545
14227
|
}
|
|
13546
14228
|
__name(resumeNeverOpened, "resumeNeverOpened");
|
|
14229
|
+
function recognizeModelUsageLimit(model, io = {}) {
|
|
14230
|
+
if (model.status === "succeeded" || model.status === "timeout") return null;
|
|
14231
|
+
const now = io.usageLimitClock?.now() ?? Date.now();
|
|
14232
|
+
const timeZone = io.usageLimitClock?.timeZone ?? localTimeZone();
|
|
14233
|
+
const options = { provider: model.provider, now, ...timeZone ? { timeZone } : {} };
|
|
14234
|
+
return detectFleetModelUsageLimit(model.summary, options) ?? detectFleetModelUsageLimit(model.outputTail, { ...options, strict: true });
|
|
14235
|
+
}
|
|
14236
|
+
__name(recognizeModelUsageLimit, "recognizeModelUsageLimit");
|
|
14237
|
+
function localTimeZone() {
|
|
14238
|
+
try {
|
|
14239
|
+
return Intl.DateTimeFormat().resolvedOptions().timeZone || void 0;
|
|
14240
|
+
} catch {
|
|
14241
|
+
return void 0;
|
|
14242
|
+
}
|
|
14243
|
+
}
|
|
14244
|
+
__name(localTimeZone, "localTimeZone");
|
|
13547
14245
|
function describeModelFailure(model) {
|
|
13548
14246
|
switch (model.status) {
|
|
13549
14247
|
case "timeout":
|
|
@@ -15137,10 +15835,14 @@ var USER_SELECTABLE_WORK_KINDS2 = [
|
|
|
15137
15835
|
"repo",
|
|
15138
15836
|
"app"
|
|
15139
15837
|
];
|
|
15140
|
-
var WORK_KINDS2 = [...USER_SELECTABLE_WORK_KINDS2, "company", "campaign", "default"];
|
|
15838
|
+
var WORK_KINDS2 = [...USER_SELECTABLE_WORK_KINDS2, "company", "campaign", "project", "default"];
|
|
15141
15839
|
new Set(WORK_KINDS2);
|
|
15142
15840
|
var GOAL_KINDS2 = ["metric", "delivery"];
|
|
15143
15841
|
new Set(GOAL_KINDS2);
|
|
15842
|
+
var RECORD_VISIBILITIES2 = ["private", "team", "organization"];
|
|
15843
|
+
new Set(RECORD_VISIBILITIES2);
|
|
15844
|
+
var RECORD_LIST_VIEWS2 = ["mine", "shared", "organization"];
|
|
15845
|
+
new Set(RECORD_LIST_VIEWS2);
|
|
15144
15846
|
var VERSION2 = "1.0.0";
|
|
15145
15847
|
var GTM_SKILLS2 = [
|
|
15146
15848
|
// ── research ────────────────────────────────────────────────────────
|
|
@@ -15847,6 +16549,8 @@ Object.freeze(
|
|
|
15847
16549
|
ACTION_CATEGORIES2.filter((c) => c !== "read.internal")
|
|
15848
16550
|
);
|
|
15849
16551
|
var DOMAIN_EVENT_CATALOG2 = [
|
|
16552
|
+
// EW-P2-6 — `org:write` (and the app) create Organizations.
|
|
16553
|
+
{ type: "organization.created", domain: "organization", aggregateType: "organization", phase: "P2" },
|
|
15850
16554
|
{ type: "organization.updated", domain: "organization", aggregateType: "organization", phase: "P1" },
|
|
15851
16555
|
{
|
|
15852
16556
|
type: "organization.member.added",
|
|
@@ -15893,10 +16597,10 @@ var DOMAIN_EVENT_CATALOG2 = [
|
|
|
15893
16597
|
{ type: "mission.updated", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15894
16598
|
{ type: "mission.status_changed", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15895
16599
|
{ type: "mission.deleted", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15896
|
-
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15897
|
-
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15898
|
-
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
15899
|
-
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
16600
|
+
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
16601
|
+
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
16602
|
+
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
16603
|
+
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
15900
16604
|
{ type: "goal.created", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
15901
16605
|
{ type: "goal.updated", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
15902
16606
|
{ type: "goal.status_changed", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
@@ -16443,6 +17147,7 @@ var LocalWorkspacePlugin = class _LocalWorkspacePlugin {
|
|
|
16443
17147
|
);
|
|
16444
17148
|
const dirty = status.stdout.trim().length > 0;
|
|
16445
17149
|
if (dirty) {
|
|
17150
|
+
const commitMessage = rewriteCommitMessage(opts, status.stdout);
|
|
16446
17151
|
const identity = opts.identity;
|
|
16447
17152
|
await this.gitOrThrow(
|
|
16448
17153
|
[
|
|
@@ -16454,7 +17159,7 @@ var LocalWorkspacePlugin = class _LocalWorkspacePlugin {
|
|
|
16454
17159
|
...opts.hardenRepoHooks ? ["--no-verify"] : [],
|
|
16455
17160
|
...identity ? [`--author=${identity.authorName} <${identity.authorEmail}>`] : [],
|
|
16456
17161
|
"-m",
|
|
16457
|
-
|
|
17162
|
+
commitMessage
|
|
16458
17163
|
],
|
|
16459
17164
|
dir,
|
|
16460
17165
|
opts.auth,
|
|
@@ -17697,6 +18402,31 @@ function isSafeCloneArgument(url) {
|
|
|
17697
18402
|
return !parsed.hostname.startsWith("-");
|
|
17698
18403
|
}
|
|
17699
18404
|
__name(isSafeCloneArgument, "isSafeCloneArgument");
|
|
18405
|
+
function parsePorcelainPaths(stdout) {
|
|
18406
|
+
const paths = [];
|
|
18407
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
18408
|
+
if (line.length < 4) continue;
|
|
18409
|
+
let path2 = line.slice(3);
|
|
18410
|
+
const arrow = path2.lastIndexOf(" -> ");
|
|
18411
|
+
if (arrow !== -1) path2 = path2.slice(arrow + 4);
|
|
18412
|
+
if (path2.startsWith('"') && path2.endsWith('"') && path2.length >= 2) {
|
|
18413
|
+
path2 = path2.slice(1, -1).replace(/\\(.)/g, "$1");
|
|
18414
|
+
}
|
|
18415
|
+
if (path2) paths.push(path2);
|
|
18416
|
+
}
|
|
18417
|
+
return paths;
|
|
18418
|
+
}
|
|
18419
|
+
__name(parsePorcelainPaths, "parsePorcelainPaths");
|
|
18420
|
+
function rewriteCommitMessage(opts, porcelain) {
|
|
18421
|
+
if (typeof opts.commitMessageFor !== "function") return opts.commitMessage;
|
|
18422
|
+
try {
|
|
18423
|
+
const next = opts.commitMessageFor(parsePorcelainPaths(porcelain), opts.commitMessage);
|
|
18424
|
+
return typeof next === "string" && next.trim() && !/[\0\r]/.test(next) ? next : opts.commitMessage;
|
|
18425
|
+
} catch {
|
|
18426
|
+
return opts.commitMessage;
|
|
18427
|
+
}
|
|
18428
|
+
}
|
|
18429
|
+
__name(rewriteCommitMessage, "rewriteCommitMessage");
|
|
17700
18430
|
|
|
17701
18431
|
// src/core/workspaces/push-credential.ts
|
|
17702
18432
|
var PUSH_CREDENTIAL_REVOKE_TIMEOUT_MS = 5e3;
|
|
@@ -18615,6 +19345,11 @@ var FleetTaskWorkspaceProvisioner = class {
|
|
|
18615
19345
|
// pool config. The credentialed push already does this; the flag
|
|
18616
19346
|
// extends it to the commit.
|
|
18617
19347
|
hardenRepoHooks: true,
|
|
19348
|
+
// Fleet commit messages: once the change is staged, a
|
|
19349
|
+
// docs-only change commits as `docs`, a tests-only one as
|
|
19350
|
+
// `test`, whatever the Task title implied. Only the type token
|
|
19351
|
+
// of the subject changes; the body and trailers are untouched.
|
|
19352
|
+
commitMessageFor: /* @__PURE__ */ __name((changedPaths, message) => retypeFleetCommitMessage(message, changedPaths), "commitMessageFor"),
|
|
18618
19353
|
...signal ? { signal } : {},
|
|
18619
19354
|
...opts.publishFence ? { publishFence: opts.publishFence } : {},
|
|
18620
19355
|
...authorization.identity ? { identity: authorization.identity } : {},
|
|
@@ -19451,10 +20186,14 @@ var USER_SELECTABLE_WORK_KINDS3 = [
|
|
|
19451
20186
|
"repo",
|
|
19452
20187
|
"app"
|
|
19453
20188
|
];
|
|
19454
|
-
var WORK_KINDS3 = [...USER_SELECTABLE_WORK_KINDS3, "company", "campaign", "default"];
|
|
20189
|
+
var WORK_KINDS3 = [...USER_SELECTABLE_WORK_KINDS3, "company", "campaign", "project", "default"];
|
|
19455
20190
|
new Set(WORK_KINDS3);
|
|
19456
20191
|
var GOAL_KINDS3 = ["metric", "delivery"];
|
|
19457
20192
|
new Set(GOAL_KINDS3);
|
|
20193
|
+
var RECORD_VISIBILITIES3 = ["private", "team", "organization"];
|
|
20194
|
+
new Set(RECORD_VISIBILITIES3);
|
|
20195
|
+
var RECORD_LIST_VIEWS3 = ["mine", "shared", "organization"];
|
|
20196
|
+
new Set(RECORD_LIST_VIEWS3);
|
|
19458
20197
|
var VERSION3 = "1.0.0";
|
|
19459
20198
|
var GTM_SKILLS3 = [
|
|
19460
20199
|
// ── research ────────────────────────────────────────────────────────
|
|
@@ -20161,6 +20900,8 @@ Object.freeze(
|
|
|
20161
20900
|
ACTION_CATEGORIES3.filter((c) => c !== "read.internal")
|
|
20162
20901
|
);
|
|
20163
20902
|
var DOMAIN_EVENT_CATALOG3 = [
|
|
20903
|
+
// EW-P2-6 — `org:write` (and the app) create Organizations.
|
|
20904
|
+
{ type: "organization.created", domain: "organization", aggregateType: "organization", phase: "P2" },
|
|
20164
20905
|
{ type: "organization.updated", domain: "organization", aggregateType: "organization", phase: "P1" },
|
|
20165
20906
|
{
|
|
20166
20907
|
type: "organization.member.added",
|
|
@@ -20207,10 +20948,10 @@ var DOMAIN_EVENT_CATALOG3 = [
|
|
|
20207
20948
|
{ type: "mission.updated", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20208
20949
|
{ type: "mission.status_changed", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20209
20950
|
{ type: "mission.deleted", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20210
|
-
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20211
|
-
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20212
|
-
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20213
|
-
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2" },
|
|
20951
|
+
{ type: "mission.goal.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
20952
|
+
{ type: "mission.goal.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "goal" },
|
|
20953
|
+
{ type: "mission.work.linked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
20954
|
+
{ type: "mission.work.unlinked", domain: "missions", aggregateType: "mission", phase: "P2", child: "work" },
|
|
20214
20955
|
{ type: "goal.created", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
20215
20956
|
{ type: "goal.updated", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
20216
20957
|
{ type: "goal.status_changed", domain: "goals", aggregateType: "goal", phase: "P2" },
|
|
@@ -21426,6 +22167,7 @@ function createNodeRuntime(config, io, options = {}) {
|
|
|
21426
22167
|
let attendedWake = null;
|
|
21427
22168
|
const holdableLanes = [];
|
|
21428
22169
|
let limitTarget = null;
|
|
22170
|
+
let throttleTarget = null;
|
|
21429
22171
|
const lifecycle = new NodeLifecycleTracker({
|
|
21430
22172
|
daemonVersion: io.version,
|
|
21431
22173
|
logger: io.logger,
|
|
@@ -21449,6 +22191,7 @@ function createNodeRuntime(config, io, options = {}) {
|
|
|
21449
22191
|
onAccepted: /* @__PURE__ */ __name((response) => {
|
|
21450
22192
|
attendedWake?.(response.pendingComputerSessions);
|
|
21451
22193
|
lifecycle.applyHeartbeat(response);
|
|
22194
|
+
throttleTarget?.(response.node?.modelThrottles);
|
|
21452
22195
|
}, "onAccepted")
|
|
21453
22196
|
};
|
|
21454
22197
|
const runtime = { client, loop: new HeartbeatLoop(loopOptions), lifecycle, cliCompat };
|
|
@@ -21490,7 +22233,11 @@ function createNodeRuntime(config, io, options = {}) {
|
|
|
21490
22233
|
// attended lane alone — never parked in (or holding a slot of) the
|
|
21491
22234
|
// work lane. Sent only under `--attend`, so a work-only node's lease
|
|
21492
22235
|
// body is exactly what it always was.
|
|
21493
|
-
...options.attendEnabled ? { excludeKinds: ["computer-session"] } : {}
|
|
22236
|
+
...options.attendEnabled ? { excludeKinds: ["computer-session"] } : {},
|
|
22237
|
+
// Fleet usage limits: the CLIs an agent-task can run with here, so a
|
|
22238
|
+
// node whose only CLI is out of quota stops leasing agent-task work
|
|
22239
|
+
// until the reset (other kinds keep leasing).
|
|
22240
|
+
modelProviders: /* @__PURE__ */ __name(() => Object.entries(options.modelCli ?? io.environment.modelCli ?? {}).filter(([, path2]) => typeof path2 === "string" && path2.length > 0).map(([provider]) => provider), "modelProviders")
|
|
21494
22241
|
});
|
|
21495
22242
|
holdableLanes.push(worker);
|
|
21496
22243
|
worker.onUpgradeRequired((hold) => lifecycle.applyLeaseRefusal(hold.minNodeVersion));
|
|
@@ -21502,6 +22249,9 @@ function createNodeRuntime(config, io, options = {}) {
|
|
|
21502
22249
|
}
|
|
21503
22250
|
}, "limitTarget");
|
|
21504
22251
|
telemetry.limits = () => worker.resourceLimits;
|
|
22252
|
+
throttleTarget = /* @__PURE__ */ __name((throttles) => {
|
|
22253
|
+
if (Array.isArray(throttles)) worker.syncModelThrottles(throttles);
|
|
22254
|
+
}, "throttleTarget");
|
|
21505
22255
|
telemetry.workerHealth = () => describeWorkerHealth(worker.getState());
|
|
21506
22256
|
const housekeeping = new NodeHousekeepingReporter({
|
|
21507
22257
|
minFreeDiskBytes: io.diskProbe ? effectiveMinFreeDiskBytes(limits) : null,
|
|
@@ -23151,10 +23901,12 @@ async function runStart(deps, options) {
|
|
|
23151
23901
|
}
|
|
23152
23902
|
const { loop, worker } = runtime;
|
|
23153
23903
|
const attendedWorker = runtime.attended?.worker ?? null;
|
|
23904
|
+
let lastLoggedBeat = null;
|
|
23154
23905
|
loop.onChange((state) => {
|
|
23155
|
-
if (state.state
|
|
23156
|
-
|
|
23157
|
-
}
|
|
23906
|
+
if (state.state !== "connected") return;
|
|
23907
|
+
if (lastLoggedBeat && lastLoggedBeat.node === state.node && lastLoggedBeat.at === state.lastHeartbeatAt) return;
|
|
23908
|
+
lastLoggedBeat = { node: state.node, at: state.lastHeartbeatAt };
|
|
23909
|
+
deps.io.logger.info(`Heartbeat accepted \u2014 node ${config.nodeId} is online`);
|
|
23158
23910
|
});
|
|
23159
23911
|
deps.out(`Starting node ${config.nodeId} \u2192 ${config.apiUrl} (every ${config.heartbeatIntervalMs / 1e3}s)`);
|
|
23160
23912
|
let requestShutdown = /* @__PURE__ */ __name(() => void 0, "requestShutdown");
|
|
@@ -23170,14 +23922,47 @@ async function runStart(deps, options) {
|
|
|
23170
23922
|
});
|
|
23171
23923
|
}
|
|
23172
23924
|
await loop.start();
|
|
23173
|
-
|
|
23174
|
-
|
|
23175
|
-
|
|
23176
|
-
|
|
23177
|
-
|
|
23178
|
-
|
|
23179
|
-
|
|
23180
|
-
|
|
23925
|
+
let shuttingDown = false;
|
|
23926
|
+
const startLanes = /* @__PURE__ */ __name(async () => {
|
|
23927
|
+
if (worker) {
|
|
23928
|
+
await worker.start();
|
|
23929
|
+
}
|
|
23930
|
+
if (attendedWorker) {
|
|
23931
|
+
await attendedWorker.start();
|
|
23932
|
+
deps.out(
|
|
23933
|
+
runtime.attended?.captureBackend ? "Live view enabled \u2014 the owner can watch this machine from the dashboard (screen and terminal)" : "Live view enabled \u2014 terminal only: no browser was found here to show a screen"
|
|
23934
|
+
);
|
|
23935
|
+
}
|
|
23936
|
+
}, "startLanes");
|
|
23937
|
+
let lanesStarted = Promise.resolve();
|
|
23938
|
+
let laneStartFailed = false;
|
|
23939
|
+
let laneStartFailure = void 0;
|
|
23940
|
+
if (worker || attendedWorker) {
|
|
23941
|
+
if (loop.getState().state === "connected") {
|
|
23942
|
+
await startLanes();
|
|
23943
|
+
} else {
|
|
23944
|
+
deps.out(
|
|
23945
|
+
"Waiting for the platform to accept a heartbeat before leasing work (its answer carries this node\u2019s limits and version verdict)"
|
|
23946
|
+
);
|
|
23947
|
+
lanesStarted = startLanesAfterAcceptedHeartbeat(
|
|
23948
|
+
loop,
|
|
23949
|
+
async () => {
|
|
23950
|
+
deps.io.logger.info("Heartbeat accepted \u2014 starting the worker lanes");
|
|
23951
|
+
await startLanes();
|
|
23952
|
+
},
|
|
23953
|
+
() => !shuttingDown
|
|
23954
|
+
).then((outcome) => {
|
|
23955
|
+
if (!outcome.failed) return;
|
|
23956
|
+
laneStartFailed = true;
|
|
23957
|
+
laneStartFailure = outcome.error;
|
|
23958
|
+
deps.io.logger.error(
|
|
23959
|
+
`Worker lanes failed to start after the first accepted heartbeat: ${describeError4(
|
|
23960
|
+
outcome.error
|
|
23961
|
+
)} \u2014 shutting down`
|
|
23962
|
+
);
|
|
23963
|
+
requestShutdown();
|
|
23964
|
+
});
|
|
23965
|
+
}
|
|
23181
23966
|
}
|
|
23182
23967
|
const reaper = worker && runtime.workspaceRoot ? startWorkspaceReaperTimer({
|
|
23183
23968
|
rootPath: runtime.workspaceRoot,
|
|
@@ -23215,8 +24000,10 @@ async function runStart(deps, options) {
|
|
|
23215
24000
|
}
|
|
23216
24001
|
await (deps.waitForShutdown?.() ?? (deps.signals ? signalled : Promise.resolve()));
|
|
23217
24002
|
reaper?.stop();
|
|
24003
|
+
shuttingDown = true;
|
|
23218
24004
|
loop.stop();
|
|
23219
24005
|
await loop.settled();
|
|
24006
|
+
await lanesStarted;
|
|
23220
24007
|
if (attendedWorker) {
|
|
23221
24008
|
for (const jobId of attendedWorker.getState().activeJobIds) {
|
|
23222
24009
|
attendedWorker.cancelJob(jobId, "This machine is shutting down");
|
|
@@ -23229,8 +24016,29 @@ async function runStart(deps, options) {
|
|
|
23229
24016
|
deps.out(`Worker host stopped (completed ${state.completed}, failed ${state.failed}).`);
|
|
23230
24017
|
}
|
|
23231
24018
|
deps.out("Stopped.");
|
|
24019
|
+
if (laneStartFailed) {
|
|
24020
|
+
throw new CliError(
|
|
24021
|
+
`Worker lanes failed to start after the first accepted heartbeat: ${describeError4(laneStartFailure)}`,
|
|
24022
|
+
EXIT_FAILURE
|
|
24023
|
+
);
|
|
24024
|
+
}
|
|
23232
24025
|
}
|
|
23233
24026
|
__name(runStart, "runStart");
|
|
24027
|
+
async function startLanesAfterAcceptedHeartbeat(loop, startLanes, shouldWait) {
|
|
24028
|
+
const accepted = await untilHeartbeatAccepted(loop, shouldWait);
|
|
24029
|
+
if (!accepted || !shouldWait()) return { started: false, failed: false };
|
|
24030
|
+
try {
|
|
24031
|
+
await startLanes();
|
|
24032
|
+
return { started: true, failed: false };
|
|
24033
|
+
} catch (error) {
|
|
24034
|
+
return { started: false, failed: true, error };
|
|
24035
|
+
}
|
|
24036
|
+
}
|
|
24037
|
+
__name(startLanesAfterAcceptedHeartbeat, "startLanesAfterAcceptedHeartbeat");
|
|
24038
|
+
function describeError4(error) {
|
|
24039
|
+
return error instanceof Error ? error.message : String(error);
|
|
24040
|
+
}
|
|
24041
|
+
__name(describeError4, "describeError");
|
|
23234
24042
|
function parseConcurrency(raw) {
|
|
23235
24043
|
if (raw === void 0) return void 0;
|
|
23236
24044
|
const parsed = Number.parseInt(raw, 10);
|
|
@@ -23897,7 +24705,7 @@ function createLogger(options = {}) {
|
|
|
23897
24705
|
__name(createLogger, "createLogger");
|
|
23898
24706
|
|
|
23899
24707
|
// src/version.ts
|
|
23900
|
-
var NODE_APP_VERSION = "0.2.
|
|
24708
|
+
var NODE_APP_VERSION = "0.2.5";
|
|
23901
24709
|
|
|
23902
24710
|
// src/cli.ts
|
|
23903
24711
|
function buildDeps() {
|