@ashlr/stack-core 0.3.6 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/_harness.ts +21 -14
- package/src/__tests__/fixtures/stack-manifest.v1.json +3 -3
- package/src/__tests__/fleet-capability-matrix.test.ts +357 -0
- package/src/__tests__/fleet-job-run.test.ts +201 -0
- package/src/__tests__/http.test.ts +53 -0
- package/src/__tests__/phantom-wire.test.ts +52 -0
- package/src/ai/phantom-wire-repeated-unknown-placeholder.test.ts +0 -3
- package/src/fleet/capability-matrix.ts +378 -0
- package/src/fleet/job-run.ts +190 -0
- package/src/http.ts +8 -15
- package/src/index.ts +2 -0
- package/src/providers/grafana.ts +0 -1
- package/src/stack-manifest.ts +3 -3
|
@@ -6430,3 +6430,55 @@ describe("webhookStubFor — arbitrary replacement with leading and trailing spa
|
|
|
6430
6430
|
).toBe(plain);
|
|
6431
6431
|
});
|
|
6432
6432
|
});
|
|
6433
|
+
|
|
6434
|
+
describe("webhookStubFor — $1 in replacement text (original PR 1023)", () => {
|
|
6435
|
+
const count = (s: string, needle: string) => s.split(needle).length - 1;
|
|
6436
|
+
|
|
6437
|
+
it("preserves $1 literally at every expansion site when it appears in a provider name", () => {
|
|
6438
|
+
// $1 is a replacement directive for String.prototype.replace/replaceAll:
|
|
6439
|
+
// it splices the first capture group. Template-literal interpolation does
|
|
6440
|
+
// NOT interpret $1, so provider names carrying it must land in the stub
|
|
6441
|
+
// as inert characters — exactly the bytes the caller passed in.
|
|
6442
|
+
const name = "svc$1pay";
|
|
6443
|
+
const stub = webhookStubFor(name);
|
|
6444
|
+
|
|
6445
|
+
// The name appears literally at the two case-preserving sites.
|
|
6446
|
+
expect(stub).toContain(`webhook handler stub for ${name}.`);
|
|
6447
|
+
expect(stub).toContain(`forge a ${name} webhook.`);
|
|
6448
|
+
|
|
6449
|
+
// At the handler name site the first character is upper-cased; $ is
|
|
6450
|
+
// ASCII so toUpperCase is the identity, leaving $1 intact.
|
|
6451
|
+
expect(stub).toContain("handleSvc$1payWebhook(req: Request)");
|
|
6452
|
+
|
|
6453
|
+
// At the reveal sites the entire name is upper-cased; $1 is
|
|
6454
|
+
// toUpperCase-invariant, so it survives unchanged.
|
|
6455
|
+
expect(count(stub, "phantom reveal SVC$1PAY_WEBHOOK_SECRET")).toBe(2);
|
|
6456
|
+
|
|
6457
|
+
// Dollar census: the name carries exactly one $ (in $1), and it appears
|
|
6458
|
+
// at five sites in the stub. No $ was dropped by $$ collapse and none
|
|
6459
|
+
// was minted by capture-group substitution.
|
|
6460
|
+
expect(count(stub, "$")).toBe(5);
|
|
6461
|
+
expect(count(stub, "$1")).toBe(5);
|
|
6462
|
+
|
|
6463
|
+
// No ${ opener leaked in — that would indicate the template was parsed
|
|
6464
|
+
// as a replace pattern and $1 was interpreted as a capture reference.
|
|
6465
|
+
expect(stub).not.toContain("${");
|
|
6466
|
+
});
|
|
6467
|
+
|
|
6468
|
+
it("preserves a bare $1 provider name through the capitalize path", () => {
|
|
6469
|
+
// $1 as the entire name exercises the capitalize() function: the first
|
|
6470
|
+
// codepoint is $, which toUpperCase leaves as $. The remaining `1` is
|
|
6471
|
+
// never touched. The name stays $1 at every expansion site.
|
|
6472
|
+
const name = "$1";
|
|
6473
|
+
const stub = webhookStubFor(name);
|
|
6474
|
+
|
|
6475
|
+
expect(stub).toContain(`webhook handler stub for ${name}.`);
|
|
6476
|
+
expect(stub).toContain(`forge a ${name} webhook.`);
|
|
6477
|
+
// capitalize("$1") = "$1" ($ is already uppercase, 1 stays 1)
|
|
6478
|
+
expect(stub).toContain("handle$1Webhook(req: Request)");
|
|
6479
|
+
expect(count(stub, "phantom reveal $1_WEBHOOK_SECRET")).toBe(2);
|
|
6480
|
+
|
|
6481
|
+
// Each $1 is a standalone $ at its own site — five total.
|
|
6482
|
+
expect(count(stub, "$1")).toBe(5);
|
|
6483
|
+
});
|
|
6484
|
+
});
|
|
@@ -1,7 +1,4 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it, setDefaultTimeout } from "bun:test";
|
|
2
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
2
|
import { type Harness, setupFakePhantom } from "../__tests__/_harness.ts";
|
|
6
3
|
import { webhookStubFor } from "./phantom-wire.ts";
|
|
7
4
|
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* capability-matrix.ts — resource-agnostic job→resource matching for the fleet.
|
|
3
|
+
*
|
|
4
|
+
* Today dispatch lanes are provider-named (codex-cmp, claude, devin, codex-local,
|
|
5
|
+
* …) and the dispatcher picks a lane per queued job. The end-state binds fleet
|
|
6
|
+
* roles to whatever resource has the capability, quota and availability — this
|
|
7
|
+
* module is that matching layer. It is pure: callers assemble `FleetResource`
|
|
8
|
+
* rows from fleet state (slots.json, journals, capabilities.json, models.json)
|
|
9
|
+
* and this file answers "which resources can serve this requirement" without
|
|
10
|
+
* ever branching on a provider or lane name.
|
|
11
|
+
*
|
|
12
|
+
* Vocabulary:
|
|
13
|
+
* - Roles (closed set): the four fleet roles a job can require.
|
|
14
|
+
* - Capabilities (open, validated): orthogonal execution properties a job may
|
|
15
|
+
* additionally require; FLEET_CAPABILITIES seeds the known tags.
|
|
16
|
+
* - Tiers: open, validated strings (the dispatcher uses frontier/work/mid/fast).
|
|
17
|
+
*
|
|
18
|
+
* Default capability profile: a resource with no capabilities.json declaration
|
|
19
|
+
* serves the `implementer` role on every tier — the same contract dispatchd
|
|
20
|
+
* applies today, where any lane can launch a coding job. A declaration can
|
|
21
|
+
* narrow roles (`roles: []` serves nothing) or widen them (planner, reviewer,
|
|
22
|
+
* release-verifier need explicit declaration).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export const FLEET_ROLES = ["planner", "implementer", "reviewer", "release-verifier"] as const;
|
|
26
|
+
export type FleetRole = (typeof FLEET_ROLES)[number];
|
|
27
|
+
|
|
28
|
+
/** Seed capability tags; the vocabulary is open — any well-formed tag is valid. */
|
|
29
|
+
export const FLEET_CAPABILITIES = [
|
|
30
|
+
/** Model inference runs on this host; loopback-only serving, no provider quota. */
|
|
31
|
+
"local-inference",
|
|
32
|
+
/** May run the publish path (fork-only pushes through publish.sh). */
|
|
33
|
+
"publish",
|
|
34
|
+
/** Can observe CI and repository state (gh or Locus-scoped reads). */
|
|
35
|
+
"ci-watch",
|
|
36
|
+
] as const;
|
|
37
|
+
|
|
38
|
+
const ROLE_SET: ReadonlySet<string> = new Set(FLEET_ROLES);
|
|
39
|
+
const RESOURCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
40
|
+
const TAG_RE = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
41
|
+
const TIER_RE = /^[a-z0-9][a-z0-9_-]{0,31}$/;
|
|
42
|
+
|
|
43
|
+
export const isFleetRole = (value: unknown): value is FleetRole =>
|
|
44
|
+
typeof value === "string" && ROLE_SET.has(value);
|
|
45
|
+
|
|
46
|
+
export const isResourceId = (value: unknown): value is string =>
|
|
47
|
+
typeof value === "string" && RESOURCE_ID_RE.test(value);
|
|
48
|
+
|
|
49
|
+
const isTag = (value: unknown): value is string => typeof value === "string" && TAG_RE.test(value);
|
|
50
|
+
const isTier = (value: unknown): value is string =>
|
|
51
|
+
typeof value === "string" && TIER_RE.test(value);
|
|
52
|
+
|
|
53
|
+
/** Well-formed capability tag (`local-inference`, `publish`, …); vocabulary is open. */
|
|
54
|
+
export const isFleetCapability = isTag;
|
|
55
|
+
/** Well-formed tier name (`frontier`, `work`, `mid`, `fast`, …); vocabulary is open. */
|
|
56
|
+
export const isFleetTier = isTier;
|
|
57
|
+
|
|
58
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
59
|
+
value !== null && typeof value === "object" && !Array.isArray(value);
|
|
60
|
+
|
|
61
|
+
/** Roles a resource serves when nothing narrows them: implementer only. */
|
|
62
|
+
export const DEFAULT_RESOURCE_ROLES: readonly FleetRole[] = ["implementer"];
|
|
63
|
+
|
|
64
|
+
/** Per-resource quota/availability projection (slots.json + journal counts). */
|
|
65
|
+
export interface FleetResourceQuota {
|
|
66
|
+
/** Operating launch bound (slots.json target); null when unknown. */
|
|
67
|
+
target: number | null;
|
|
68
|
+
ceiling: number | null;
|
|
69
|
+
hardCap: number | null;
|
|
70
|
+
/** Operator or autobackoff hold. */
|
|
71
|
+
held: boolean;
|
|
72
|
+
/** Observed running jobs for this resource; null when journals are unreadable. */
|
|
73
|
+
running: number | null;
|
|
74
|
+
/** Observed queued jobs for this resource; null when journals are unreadable. */
|
|
75
|
+
queued: number | null;
|
|
76
|
+
/** Provider quota reading from slots.json when present; informational only —
|
|
77
|
+
* the dispatcher does not launch-block on it, so neither does matching. */
|
|
78
|
+
quotaRemaining: number | null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** What capabilities.json may declare for one resource. */
|
|
82
|
+
export interface CapabilityDeclaration {
|
|
83
|
+
/** null = default (implementer); [] = serves no role. */
|
|
84
|
+
roles: readonly FleetRole[] | null;
|
|
85
|
+
capabilities: readonly string[];
|
|
86
|
+
/** null = every tier; otherwise the tiers this resource serves. */
|
|
87
|
+
tiers: readonly string[] | null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** A schedulable resource after declarations and live quota are merged. */
|
|
91
|
+
export interface FleetResource {
|
|
92
|
+
id: string;
|
|
93
|
+
/** True when capabilities.json declared this resource. */
|
|
94
|
+
declared: boolean;
|
|
95
|
+
/** Effective roles (declaration or the implementer default). */
|
|
96
|
+
roles: readonly FleetRole[];
|
|
97
|
+
capabilities: readonly string[];
|
|
98
|
+
tiers: readonly string[] | null;
|
|
99
|
+
/** null = no slot data; capacity checks then report headroom null, never block. */
|
|
100
|
+
quota: FleetResourceQuota | null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** What a job needs: a role, optional extra capability tags, optional tier. */
|
|
104
|
+
export interface JobRequirement {
|
|
105
|
+
role: FleetRole;
|
|
106
|
+
capabilities: readonly string[];
|
|
107
|
+
tier: string | null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export interface ResourceMatch {
|
|
111
|
+
id: string;
|
|
112
|
+
eligible: boolean;
|
|
113
|
+
/** Blocking reason codes; empty iff eligible. Codes: `missing-role:<role>`,
|
|
114
|
+
* `missing-capability:<tag>`, `tier-unsupported:<tier>`, `held`,
|
|
115
|
+
* `fleet-hold`, `fleet-at-capacity`, `at-capacity`. */
|
|
116
|
+
reasons: string[];
|
|
117
|
+
/** Spare launch slots (capacity bound minus running); null when either side
|
|
118
|
+
* is unknown — unknown quota never blocks on capacity, it reports null. */
|
|
119
|
+
headroom: number | null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export interface MatchResult {
|
|
123
|
+
requirement: JobRequirement;
|
|
124
|
+
/** Eligible resources first (headroom desc, id asc), then ineligible by id. */
|
|
125
|
+
matches: ResourceMatch[];
|
|
126
|
+
/** Ids of eligible resources in match order. */
|
|
127
|
+
eligible: string[];
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface MatchOptions {
|
|
131
|
+
/** HOLD file present at the queue dir — blocks every resource. */
|
|
132
|
+
fleetHold?: boolean;
|
|
133
|
+
/** Fleet-wide running total and hard cap; blocks all only when certainly full. */
|
|
134
|
+
fleet?: { running: number | null; hardCap: number | null };
|
|
135
|
+
/** Cap on returned matches (eligible ordering preserved). Default unlimited. */
|
|
136
|
+
limit?: number;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export interface CapabilityDocument {
|
|
140
|
+
schema: "fleet.capabilities.v1";
|
|
141
|
+
resources: ReadonlyMap<string, CapabilityDeclaration>;
|
|
142
|
+
/** Resource entries skipped for invalid shape (they fall back to defaults). */
|
|
143
|
+
dropped: number;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export type CapabilityDocumentResult =
|
|
147
|
+
| { status: "ok"; document: CapabilityDocument; error: null }
|
|
148
|
+
| { status: "invalid"; document: null; error: string };
|
|
149
|
+
|
|
150
|
+
const nonNegInt = (value: unknown): number | null =>
|
|
151
|
+
typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
152
|
+
|
|
153
|
+
function parseStringList(value: unknown, valid: (v: unknown) => boolean): string[] | null {
|
|
154
|
+
if (!Array.isArray(value) || value.some((v) => !valid(v))) return null;
|
|
155
|
+
return [...new Set(value as string[])];
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Parse a capabilities.json document. Strict about shape, lenient about scope:
|
|
160
|
+
* unknown keys are ignored, but a resource entry with an invalid roles /
|
|
161
|
+
* capabilities / tiers field is dropped whole and counted in `dropped` — an
|
|
162
|
+
* invalid declaration must never silently narrow or widen a resource.
|
|
163
|
+
*/
|
|
164
|
+
export function parseCapabilityDocument(text: string): CapabilityDocumentResult {
|
|
165
|
+
let root: unknown;
|
|
166
|
+
try {
|
|
167
|
+
root = JSON.parse(text);
|
|
168
|
+
} catch {
|
|
169
|
+
return { status: "invalid", document: null, error: "not valid JSON" };
|
|
170
|
+
}
|
|
171
|
+
if (!isRecord(root)) return { status: "invalid", document: null, error: "not a JSON object" };
|
|
172
|
+
if (root.schema !== "fleet.capabilities.v1")
|
|
173
|
+
return { status: "invalid", document: null, error: 'expected schema "fleet.capabilities.v1"' };
|
|
174
|
+
if (root.resources !== undefined && !isRecord(root.resources))
|
|
175
|
+
return { status: "invalid", document: null, error: '"resources" must be an object' };
|
|
176
|
+
|
|
177
|
+
const resources = new Map<string, CapabilityDeclaration>();
|
|
178
|
+
let dropped = 0;
|
|
179
|
+
for (const [id, value] of Object.entries(isRecord(root.resources) ? root.resources : {})) {
|
|
180
|
+
if (!isResourceId(id) || !isRecord(value)) {
|
|
181
|
+
dropped++;
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
let roles: FleetRole[] | null = null;
|
|
185
|
+
if (value.roles !== undefined) {
|
|
186
|
+
const parsed = parseStringList(value.roles, isFleetRole);
|
|
187
|
+
if (parsed === null) {
|
|
188
|
+
dropped++;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
roles = parsed as FleetRole[];
|
|
192
|
+
}
|
|
193
|
+
let capabilities: string[] = [];
|
|
194
|
+
if (value.capabilities !== undefined) {
|
|
195
|
+
const parsed = parseStringList(value.capabilities, isTag);
|
|
196
|
+
if (parsed === null) {
|
|
197
|
+
dropped++;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
capabilities = parsed;
|
|
201
|
+
}
|
|
202
|
+
let tiers: string[] | null = null;
|
|
203
|
+
if (value.tiers !== undefined) {
|
|
204
|
+
const parsed = parseStringList(value.tiers, isTier);
|
|
205
|
+
if (parsed === null) {
|
|
206
|
+
dropped++;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
tiers = parsed;
|
|
210
|
+
}
|
|
211
|
+
resources.set(id, { roles, capabilities, tiers });
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
status: "ok",
|
|
215
|
+
document: { schema: "fleet.capabilities.v1", resources, dropped },
|
|
216
|
+
error: null,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Effective roles for a declaration; absent declaration or absent `roles` → implementer. */
|
|
221
|
+
export function effectiveRoles(
|
|
222
|
+
declaration: CapabilityDeclaration | undefined,
|
|
223
|
+
): readonly FleetRole[] {
|
|
224
|
+
return declaration?.roles ?? DEFAULT_RESOURCE_ROLES;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** Merge a declaration with observed quota into a matchable resource. */
|
|
228
|
+
export function buildFleetResource(
|
|
229
|
+
id: string,
|
|
230
|
+
declaration: CapabilityDeclaration | undefined,
|
|
231
|
+
quota: FleetResourceQuota | null,
|
|
232
|
+
): FleetResource {
|
|
233
|
+
return {
|
|
234
|
+
id,
|
|
235
|
+
declared: declaration !== undefined,
|
|
236
|
+
roles: effectiveRoles(declaration),
|
|
237
|
+
capabilities: declaration?.capabilities ?? [],
|
|
238
|
+
tiers: declaration?.tiers ?? null,
|
|
239
|
+
quota,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Smallest known launch bound; null when no quota field is usable. */
|
|
244
|
+
export function capacityBound(quota: FleetResourceQuota): number | null {
|
|
245
|
+
const bounds = [quota.target, quota.ceiling, quota.hardCap].filter(
|
|
246
|
+
(v): v is number => v !== null,
|
|
247
|
+
);
|
|
248
|
+
return bounds.length ? Math.min(...bounds) : null;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** capacityBound - running; null when either side is unknown. */
|
|
252
|
+
export function resourceHeadroom(quota: FleetResourceQuota | null): number | null {
|
|
253
|
+
if (quota === null || quota.running === null) return null;
|
|
254
|
+
const bound = capacityBound(quota);
|
|
255
|
+
return bound === null ? null : Math.max(0, bound - quota.running);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Match one job requirement against resources. Provider- and lane-name
|
|
260
|
+
* agnostic: only declared roles, capability tags, tier coverage, holds and
|
|
261
|
+
* observed quota decide eligibility.
|
|
262
|
+
*/
|
|
263
|
+
export function matchRequirement(
|
|
264
|
+
requirement: JobRequirement,
|
|
265
|
+
resources: readonly FleetResource[],
|
|
266
|
+
options: MatchOptions = {},
|
|
267
|
+
): MatchResult {
|
|
268
|
+
const fleetFull =
|
|
269
|
+
options.fleet?.hardCap != null &&
|
|
270
|
+
options.fleet.running != null &&
|
|
271
|
+
options.fleet.running >= options.fleet.hardCap;
|
|
272
|
+
const matches: ResourceMatch[] = resources.map((resource) => {
|
|
273
|
+
const reasons: string[] = [];
|
|
274
|
+
if (!resource.roles.includes(requirement.role))
|
|
275
|
+
reasons.push(`missing-role:${requirement.role}`);
|
|
276
|
+
for (const tag of requirement.capabilities)
|
|
277
|
+
if (!resource.capabilities.includes(tag)) reasons.push(`missing-capability:${tag}`);
|
|
278
|
+
if (
|
|
279
|
+
requirement.tier !== null &&
|
|
280
|
+
resource.tiers !== null &&
|
|
281
|
+
!resource.tiers.includes(requirement.tier)
|
|
282
|
+
)
|
|
283
|
+
reasons.push(`tier-unsupported:${requirement.tier}`);
|
|
284
|
+
if (options.fleetHold === true) reasons.push("fleet-hold");
|
|
285
|
+
if (fleetFull) reasons.push("fleet-at-capacity");
|
|
286
|
+
if (resource.quota?.held === true) reasons.push("held");
|
|
287
|
+
const headroom = resourceHeadroom(resource.quota);
|
|
288
|
+
if (headroom === 0) reasons.push("at-capacity");
|
|
289
|
+
return { id: resource.id, eligible: reasons.length === 0, reasons, headroom };
|
|
290
|
+
});
|
|
291
|
+
matches.sort((a, b) => {
|
|
292
|
+
if (a.eligible !== b.eligible) return a.eligible ? -1 : 1;
|
|
293
|
+
if (a.eligible) {
|
|
294
|
+
// Eligible: most headroom first; unknown headroom sorts last; id breaks ties.
|
|
295
|
+
if (a.headroom !== b.headroom) {
|
|
296
|
+
if (a.headroom === null) return 1;
|
|
297
|
+
if (b.headroom === null) return -1;
|
|
298
|
+
return b.headroom - a.headroom;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return a.id.localeCompare(b.id);
|
|
302
|
+
});
|
|
303
|
+
const limited = options.limit === undefined ? matches : matches.slice(0, options.limit);
|
|
304
|
+
return {
|
|
305
|
+
requirement,
|
|
306
|
+
matches: limited,
|
|
307
|
+
eligible: limited.filter((m) => m.eligible).map((m) => m.id),
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// ---------------------------------------------------------------------------
|
|
312
|
+
// models.json — flat `lane[.tier]` → model map; resolution mirrors dispatchd.
|
|
313
|
+
// ---------------------------------------------------------------------------
|
|
314
|
+
|
|
315
|
+
const MODEL_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
316
|
+
const MODEL_RE = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$/;
|
|
317
|
+
|
|
318
|
+
export interface ModelMap {
|
|
319
|
+
map: ReadonlyMap<string, string>;
|
|
320
|
+
dropped: number;
|
|
321
|
+
/** "invalid" when the text is not a JSON object; entries may still be dropped from a valid doc. */
|
|
322
|
+
status: "ok" | "invalid";
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Parse ~/.ashlr-runs/models.json: `{ "<resource>[.<tier>]": "<model>" }`. */
|
|
326
|
+
export function parseModelMap(text: string): ModelMap {
|
|
327
|
+
const map = new Map<string, string>();
|
|
328
|
+
let dropped = 0;
|
|
329
|
+
let root: unknown;
|
|
330
|
+
try {
|
|
331
|
+
root = JSON.parse(text);
|
|
332
|
+
} catch {
|
|
333
|
+
return { map, dropped: 1, status: "invalid" };
|
|
334
|
+
}
|
|
335
|
+
if (!isRecord(root)) return { map, dropped: 1, status: "invalid" };
|
|
336
|
+
for (const [key, value] of Object.entries(root)) {
|
|
337
|
+
const dot = key.indexOf(".");
|
|
338
|
+
const lane = dot < 0 ? key : key.slice(0, dot);
|
|
339
|
+
const tier = dot < 0 ? null : key.slice(dot + 1);
|
|
340
|
+
if (
|
|
341
|
+
!MODEL_KEY_RE.test(lane) ||
|
|
342
|
+
(tier !== null && !isTier(tier)) ||
|
|
343
|
+
typeof value !== "string" ||
|
|
344
|
+
!MODEL_RE.test(value.trim())
|
|
345
|
+
) {
|
|
346
|
+
dropped++;
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
map.set(key, value.trim());
|
|
350
|
+
}
|
|
351
|
+
return { map, dropped, status: "ok" };
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** The map family dispatchd uses: `codex` for codex-* lanes except codex-local. */
|
|
355
|
+
export const modelFamily = (resourceId: string): string =>
|
|
356
|
+
resourceId.startsWith("codex") && resourceId !== "codex-local" ? "codex" : resourceId;
|
|
357
|
+
|
|
358
|
+
/** First non-empty of `<id>.<tier>`, `<id>`, `<family>.<tier>`, `<family>`. */
|
|
359
|
+
export function resolveModel(
|
|
360
|
+
map: ReadonlyMap<string, string>,
|
|
361
|
+
resourceId: string,
|
|
362
|
+
tier: string | null,
|
|
363
|
+
): { model: string; via: string } | null {
|
|
364
|
+
const family = modelFamily(resourceId);
|
|
365
|
+
const keys =
|
|
366
|
+
family === resourceId
|
|
367
|
+
? tier === null
|
|
368
|
+
? [resourceId]
|
|
369
|
+
: [`${resourceId}.${tier}`, resourceId]
|
|
370
|
+
: tier === null
|
|
371
|
+
? [resourceId, family]
|
|
372
|
+
: [`${resourceId}.${tier}`, resourceId, `${family}.${tier}`, family];
|
|
373
|
+
for (const key of keys) {
|
|
374
|
+
const model = map.get(key);
|
|
375
|
+
if (model) return { model, via: key };
|
|
376
|
+
}
|
|
377
|
+
return null;
|
|
378
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { appendFile, mkdir } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* job-run.ts — per-execution structured result records for fleet jobs.
|
|
6
|
+
*
|
|
7
|
+
* Motivation: stacksync execution 5d2b848b died inside `write_sheet_jobs` and
|
|
8
|
+
* the only artifact was an unstructured error line — remotely we could not
|
|
9
|
+
* tell which stage failed, how long each stage took, or how many rows were
|
|
10
|
+
* pulled vs. committed. Every staged job run now ends with one JobRunRecord:
|
|
11
|
+
* per-stage timings, rows in/out, and the last error tagged with its stage.
|
|
12
|
+
*
|
|
13
|
+
* The record is emitted as a single-line JSON log (greppable in journald or
|
|
14
|
+
* any log drain) and, when `tablePath` is set, appended as one row to a JSONL
|
|
15
|
+
* table that `readFleetJsonl` / `jq` can query. Records carry counts and a
|
|
16
|
+
* bounded, flattened error message — never row payloads.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export const JOB_RUN_SCHEMA = "fleet.job-run.v1";
|
|
20
|
+
|
|
21
|
+
/** Stacksync's fixed pipeline order: source read → row transform → sheet write. */
|
|
22
|
+
export const SYNC_STAGES = ["read_postgres", "transform", "write_sheet_jobs"] as const;
|
|
23
|
+
export type SyncStageName = (typeof SYNC_STAGES)[number];
|
|
24
|
+
|
|
25
|
+
export type StageStatus = "ok" | "error" | "skipped";
|
|
26
|
+
|
|
27
|
+
export interface StageTiming {
|
|
28
|
+
stage: string;
|
|
29
|
+
status: StageStatus;
|
|
30
|
+
/** Wall time of the stage; null when the stage never ran (skipped). */
|
|
31
|
+
durationMs: number | null;
|
|
32
|
+
/** Rows the stage consumed. */
|
|
33
|
+
rowsIn: number;
|
|
34
|
+
/** Rows the stage produced; on a failed write this is the partial commit count. */
|
|
35
|
+
rowsOut: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface JobRunError {
|
|
39
|
+
stage: string;
|
|
40
|
+
message: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface JobRunRecord {
|
|
44
|
+
schema: typeof JOB_RUN_SCHEMA;
|
|
45
|
+
job: string;
|
|
46
|
+
executionId: string;
|
|
47
|
+
/** Sync direction for two-way jobs, e.g. "postgres->sheets"; null when unset. */
|
|
48
|
+
direction: string | null;
|
|
49
|
+
startedAt: string;
|
|
50
|
+
finishedAt: string;
|
|
51
|
+
durationMs: number;
|
|
52
|
+
status: "ok" | "error";
|
|
53
|
+
/** Rows that entered the pipeline: the first stage's input, or its output when
|
|
54
|
+
* it is a source stage (read_postgres reports only rowsOut). */
|
|
55
|
+
rowsIn: number;
|
|
56
|
+
/** Rows the pipeline emitted: the last attempted stage's output — partial on failure. */
|
|
57
|
+
rowsOut: number;
|
|
58
|
+
stages: StageTiming[];
|
|
59
|
+
lastError: JobRunError | null;
|
|
60
|
+
/** Set when the JSONL table append failed; the log line is then the only copy. */
|
|
61
|
+
persistError: string | null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Mutable counters handed to a stage; partial values survive a throw. */
|
|
65
|
+
export interface StageCounts {
|
|
66
|
+
rowsIn: number;
|
|
67
|
+
rowsOut: number;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface JobStage {
|
|
71
|
+
name: string;
|
|
72
|
+
run: (counts: StageCounts) => void | Promise<void>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface RunJobOptions {
|
|
76
|
+
stages: readonly JobStage[];
|
|
77
|
+
/** Job name stamped on the record. Default "stacksync". */
|
|
78
|
+
job?: string;
|
|
79
|
+
/** Caller-assigned execution id (e.g. "5d2b848b"); defaults to a random 8-hex id. */
|
|
80
|
+
executionId?: string;
|
|
81
|
+
/** Sync direction label, e.g. "postgres->sheets". */
|
|
82
|
+
direction?: string;
|
|
83
|
+
/** JSONL table path; when set the record is appended as one line. */
|
|
84
|
+
tablePath?: string;
|
|
85
|
+
/** Log-line sink; default writes one JSON line to stdout. Pass `false` to silence. */
|
|
86
|
+
logLine?: ((line: string) => void) | false;
|
|
87
|
+
/** Wall clock for ISO stamps and durations (ms); default Date.now. */
|
|
88
|
+
now?: () => number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Cap on recorded error text — provider error bodies can be kilobytes. */
|
|
92
|
+
const MAX_ERROR_MESSAGE = 500;
|
|
93
|
+
|
|
94
|
+
function errorMessage(error: unknown): string {
|
|
95
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
96
|
+
const flat = raw.replace(/\s+/g, " ").trim();
|
|
97
|
+
return flat.length > MAX_ERROR_MESSAGE ? `${flat.slice(0, MAX_ERROR_MESSAGE)}...` : flat;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Junk counts (negative, non-finite) record as 0 — a record must never carry NaN. */
|
|
101
|
+
const rowCount = (value: number): number =>
|
|
102
|
+
Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
103
|
+
|
|
104
|
+
const defaultLogLine = (line: string): void => console.log(line);
|
|
105
|
+
|
|
106
|
+
/** Append one record to a JSONL table, creating the parent directory as needed. */
|
|
107
|
+
export async function appendJobRunRecord(path: string, record: JobRunRecord): Promise<void> {
|
|
108
|
+
await mkdir(dirname(path), { recursive: true });
|
|
109
|
+
await appendFile(path, `${JSON.stringify(record)}\n`, "utf8");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Run `stages` in order and return the execution record. The record — not a
|
|
114
|
+
* throw — is the outcome: callers branch on `record.status` /
|
|
115
|
+
* `record.lastError`. A stage that throws ends the run; the failing stage's
|
|
116
|
+
* timing and partial counts are kept, later stages are marked "skipped", and
|
|
117
|
+
* the record is still emitted so a remote run leaves evidence.
|
|
118
|
+
*/
|
|
119
|
+
export async function runJobStages(options: RunJobOptions): Promise<JobRunRecord> {
|
|
120
|
+
const now = options.now ?? Date.now;
|
|
121
|
+
const startMs = now();
|
|
122
|
+
const timings: StageTiming[] = [];
|
|
123
|
+
let lastError: JobRunError | null = null;
|
|
124
|
+
|
|
125
|
+
for (const stage of options.stages) {
|
|
126
|
+
if (lastError !== null) {
|
|
127
|
+
timings.push({
|
|
128
|
+
stage: stage.name,
|
|
129
|
+
status: "skipped",
|
|
130
|
+
durationMs: null,
|
|
131
|
+
rowsIn: 0,
|
|
132
|
+
rowsOut: 0,
|
|
133
|
+
});
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const counts: StageCounts = { rowsIn: 0, rowsOut: 0 };
|
|
137
|
+
const stageStart = now();
|
|
138
|
+
try {
|
|
139
|
+
await stage.run(counts);
|
|
140
|
+
timings.push({
|
|
141
|
+
stage: stage.name,
|
|
142
|
+
status: "ok",
|
|
143
|
+
durationMs: Math.max(0, Math.round(now() - stageStart)),
|
|
144
|
+
rowsIn: rowCount(counts.rowsIn),
|
|
145
|
+
rowsOut: rowCount(counts.rowsOut),
|
|
146
|
+
});
|
|
147
|
+
} catch (error) {
|
|
148
|
+
timings.push({
|
|
149
|
+
stage: stage.name,
|
|
150
|
+
status: "error",
|
|
151
|
+
durationMs: Math.max(0, Math.round(now() - stageStart)),
|
|
152
|
+
rowsIn: rowCount(counts.rowsIn),
|
|
153
|
+
rowsOut: rowCount(counts.rowsOut),
|
|
154
|
+
});
|
|
155
|
+
lastError = { stage: stage.name, message: errorMessage(error) };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const endMs = now();
|
|
160
|
+
const attempted = timings.filter((t) => t.status !== "skipped");
|
|
161
|
+
const first = attempted[0];
|
|
162
|
+
const last = attempted[attempted.length - 1];
|
|
163
|
+
const record: JobRunRecord = {
|
|
164
|
+
schema: JOB_RUN_SCHEMA,
|
|
165
|
+
job: options.job ?? "stacksync",
|
|
166
|
+
executionId: options.executionId ?? crypto.randomUUID().slice(0, 8),
|
|
167
|
+
direction: options.direction ?? null,
|
|
168
|
+
startedAt: new Date(startMs).toISOString(),
|
|
169
|
+
finishedAt: new Date(endMs).toISOString(),
|
|
170
|
+
durationMs: Math.max(0, Math.round(endMs - startMs)),
|
|
171
|
+
status: lastError === null ? "ok" : "error",
|
|
172
|
+
rowsIn: first ? first.rowsIn || first.rowsOut : 0,
|
|
173
|
+
rowsOut: last?.rowsOut ?? 0,
|
|
174
|
+
stages: timings,
|
|
175
|
+
lastError,
|
|
176
|
+
persistError: null,
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
if (options.tablePath) {
|
|
180
|
+
try {
|
|
181
|
+
await appendJobRunRecord(options.tablePath, record);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
record.persistError = errorMessage(error);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const sink = options.logLine === undefined ? defaultLogLine : options.logLine;
|
|
188
|
+
if (sink) sink(JSON.stringify(record));
|
|
189
|
+
return record;
|
|
190
|
+
}
|
package/src/http.ts
CHANGED
|
@@ -56,23 +56,21 @@ export async function fetchWithRetry(
|
|
|
56
56
|
const baseDelayMs = opts.baseDelayMs ?? 400;
|
|
57
57
|
const maxDelayMs = opts.maxDelayMs ?? 8000;
|
|
58
58
|
const retryOn = opts.retryOn ?? defaultRetryOn;
|
|
59
|
-
const idempotent =
|
|
60
|
-
|
|
59
|
+
const idempotent =
|
|
60
|
+
opts.idempotent ??
|
|
61
|
+
isIdempotent(init?.method ?? (input instanceof Request ? input.method : undefined));
|
|
62
|
+
const requestSignal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
|
|
63
|
+
const mergedInit = mergeSignal({ ...init, signal: requestSignal }, opts.signal);
|
|
64
|
+
const signal = mergedInit?.signal ?? undefined;
|
|
61
65
|
|
|
62
66
|
let attempt = 0;
|
|
63
|
-
// Track the last non-ok response so a final failure returns something the
|
|
64
|
-
// caller can inspect (status / body) rather than a synthetic error.
|
|
65
|
-
let lastResponse: Response | undefined;
|
|
66
|
-
let lastError: unknown;
|
|
67
|
-
|
|
68
67
|
while (true) {
|
|
69
68
|
if (signal?.aborted) throw abortError(signal);
|
|
70
69
|
|
|
71
70
|
try {
|
|
72
|
-
const res = await fetch(input,
|
|
71
|
+
const res = await fetch(input, mergedInit);
|
|
73
72
|
if (res.ok || !idempotent || !retryOn(res)) return res;
|
|
74
73
|
|
|
75
|
-
lastResponse = res;
|
|
76
74
|
if (attempt >= retries) return res;
|
|
77
75
|
|
|
78
76
|
const waitMs = computeDelay(res, attempt, baseDelayMs, maxDelayMs);
|
|
@@ -82,19 +80,14 @@ export async function fetchWithRetry(
|
|
|
82
80
|
// Network-level failure (DNS, connection reset, fetch threw). Only retry
|
|
83
81
|
// for idempotent requests — a POST that "failed" may have actually been
|
|
84
82
|
// accepted by the remote before the connection dropped.
|
|
83
|
+
if (signal?.aborted) throw abortError(signal);
|
|
85
84
|
if (isAbortError(err)) throw err;
|
|
86
|
-
lastError = err;
|
|
87
85
|
if (!idempotent || attempt >= retries) throw err;
|
|
88
86
|
const waitMs = computeDelay(undefined, attempt, baseDelayMs, maxDelayMs);
|
|
89
87
|
await sleep(waitMs, signal);
|
|
90
88
|
attempt++;
|
|
91
89
|
}
|
|
92
90
|
}
|
|
93
|
-
|
|
94
|
-
// Unreachable — both branches above either return or throw. The assignment
|
|
95
|
-
// to lastResponse/lastError keeps TS happy if the loop ever changes shape.
|
|
96
|
-
void lastResponse;
|
|
97
|
-
void lastError;
|
|
98
91
|
}
|
|
99
92
|
|
|
100
93
|
// ---------------------------------------------------------------------------
|