@albinocrabs/o-switcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +72 -0
- package/LICENSE +21 -0
- package/README.md +361 -0
- package/dist/chunk-BTDKGS7P.js +1777 -0
- package/dist/chunk-BTDKGS7P.js.map +1 -0
- package/dist/index.cjs +2585 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +2021 -0
- package/dist/index.d.ts +2021 -0
- package/dist/index.js +835 -0
- package/dist/index.js.map +1 -0
- package/dist/plugin.cjs +1177 -0
- package/dist/plugin.cjs.map +1 -0
- package/dist/plugin.d.cts +22 -0
- package/dist/plugin.d.ts +22 -0
- package/dist/plugin.js +194 -0
- package/dist/plugin.js.map +1 -0
- package/docs/api-reference.md +286 -0
- package/docs/architecture.md +511 -0
- package/docs/examples.md +190 -0
- package/docs/getting-started.md +316 -0
- package/package.json +60 -0
- package/scripts/collect-errors.ts +159 -0
- package/scripts/corpus.jsonl +5 -0
|
@@ -0,0 +1,1777 @@
|
|
|
1
|
+
// src/config/defaults.ts
|
|
2
|
+
var DEFAULT_RETRY_BUDGET = 3;
|
|
3
|
+
var DEFAULT_FAILOVER_BUDGET = 2;
|
|
4
|
+
var DEFAULT_BACKOFF_BASE_MS = 1e3;
|
|
5
|
+
var DEFAULT_BACKOFF_MULTIPLIER = 2;
|
|
6
|
+
var DEFAULT_BACKOFF_MAX_MS = 3e4;
|
|
7
|
+
var DEFAULT_BACKOFF_JITTER = "full";
|
|
8
|
+
var DEFAULT_ROUTING_WEIGHT_HEALTH = 1;
|
|
9
|
+
var DEFAULT_ROUTING_WEIGHT_LATENCY = 0.5;
|
|
10
|
+
var DEFAULT_ROUTING_WEIGHT_FAILURE = 0.8;
|
|
11
|
+
var DEFAULT_ROUTING_WEIGHT_PRIORITY = 0.3;
|
|
12
|
+
var DEFAULT_MAX_EXPECTED_LATENCY_MS = 3e4;
|
|
13
|
+
var DEFAULT_QUEUE_LIMIT = 100;
|
|
14
|
+
var DEFAULT_GLOBAL_CONCURRENCY_LIMIT = 10;
|
|
15
|
+
var DEFAULT_BACKPRESSURE_THRESHOLD = 50;
|
|
16
|
+
var DEFAULT_CB_FAILURE_THRESHOLD = 5;
|
|
17
|
+
var DEFAULT_CB_FAILURE_RATE_THRESHOLD = 0.5;
|
|
18
|
+
var DEFAULT_CB_SLIDING_WINDOW_SIZE = 10;
|
|
19
|
+
var DEFAULT_CB_HALF_OPEN_AFTER_MS = 3e4;
|
|
20
|
+
var DEFAULT_CB_HALF_OPEN_MAX_PROBES = 1;
|
|
21
|
+
var DEFAULT_CB_SUCCESS_THRESHOLD = 2;
|
|
22
|
+
var DEFAULT_RETRY = 3;
|
|
23
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
24
|
+
|
|
25
|
+
// src/config/schema.ts
|
|
26
|
+
import { z } from "zod";
|
|
27
|
+
var BackoffConfigSchema = z.object({
|
|
28
|
+
base_ms: z.number().positive().default(DEFAULT_BACKOFF_BASE_MS),
|
|
29
|
+
multiplier: z.number().positive().default(DEFAULT_BACKOFF_MULTIPLIER),
|
|
30
|
+
max_ms: z.number().positive().default(DEFAULT_BACKOFF_MAX_MS),
|
|
31
|
+
jitter: z.enum(["full", "equal", "none"]).default(DEFAULT_BACKOFF_JITTER)
|
|
32
|
+
});
|
|
33
|
+
var RoutingWeightsSchema = z.object({
|
|
34
|
+
health: z.number().default(DEFAULT_ROUTING_WEIGHT_HEALTH),
|
|
35
|
+
latency: z.number().default(DEFAULT_ROUTING_WEIGHT_LATENCY),
|
|
36
|
+
failure: z.number().default(DEFAULT_ROUTING_WEIGHT_FAILURE),
|
|
37
|
+
priority: z.number().default(DEFAULT_ROUTING_WEIGHT_PRIORITY)
|
|
38
|
+
});
|
|
39
|
+
var CircuitBreakerConfigSchema = z.object({
|
|
40
|
+
failure_threshold: z.number().int().positive().default(DEFAULT_CB_FAILURE_THRESHOLD),
|
|
41
|
+
failure_rate_threshold: z.number().gt(0).lt(1).default(DEFAULT_CB_FAILURE_RATE_THRESHOLD),
|
|
42
|
+
sliding_window_size: z.number().int().positive().default(DEFAULT_CB_SLIDING_WINDOW_SIZE),
|
|
43
|
+
half_open_after_ms: z.number().positive().default(DEFAULT_CB_HALF_OPEN_AFTER_MS),
|
|
44
|
+
half_open_max_probes: z.number().int().positive().default(DEFAULT_CB_HALF_OPEN_MAX_PROBES),
|
|
45
|
+
success_threshold: z.number().int().positive().default(DEFAULT_CB_SUCCESS_THRESHOLD)
|
|
46
|
+
});
|
|
47
|
+
var TargetConfigSchema = z.object({
|
|
48
|
+
target_id: z.string().min(1),
|
|
49
|
+
provider_id: z.string().min(1),
|
|
50
|
+
profile: z.string().optional(),
|
|
51
|
+
endpoint_id: z.string().optional(),
|
|
52
|
+
capabilities: z.array(z.string()).default([]),
|
|
53
|
+
enabled: z.boolean().default(true),
|
|
54
|
+
operator_priority: z.number().int().default(0),
|
|
55
|
+
policy_tags: z.array(z.string()).default([]),
|
|
56
|
+
retry_budget: z.number().int().positive().optional(),
|
|
57
|
+
failover_budget: z.number().int().positive().optional(),
|
|
58
|
+
backoff: BackoffConfigSchema.optional(),
|
|
59
|
+
concurrency_limit: z.number().int().positive().optional(),
|
|
60
|
+
circuit_breaker: CircuitBreakerConfigSchema.optional()
|
|
61
|
+
});
|
|
62
|
+
var SwitcherConfigSchema = z.object({
|
|
63
|
+
targets: z.array(TargetConfigSchema).min(1).optional(),
|
|
64
|
+
retry: z.number().int().positive().optional(),
|
|
65
|
+
timeout: z.number().positive().optional(),
|
|
66
|
+
retry_budget: z.number().int().positive().default(DEFAULT_RETRY_BUDGET),
|
|
67
|
+
failover_budget: z.number().int().positive().default(DEFAULT_FAILOVER_BUDGET),
|
|
68
|
+
backoff: BackoffConfigSchema.default({
|
|
69
|
+
base_ms: DEFAULT_BACKOFF_BASE_MS,
|
|
70
|
+
multiplier: DEFAULT_BACKOFF_MULTIPLIER,
|
|
71
|
+
max_ms: DEFAULT_BACKOFF_MAX_MS,
|
|
72
|
+
jitter: DEFAULT_BACKOFF_JITTER
|
|
73
|
+
}),
|
|
74
|
+
deployment_mode_hint: z.enum(["plugin-only", "server-companion", "sdk-control", "auto"]).default("auto"),
|
|
75
|
+
routing_weights: RoutingWeightsSchema.default({
|
|
76
|
+
health: DEFAULT_ROUTING_WEIGHT_HEALTH,
|
|
77
|
+
latency: DEFAULT_ROUTING_WEIGHT_LATENCY,
|
|
78
|
+
failure: DEFAULT_ROUTING_WEIGHT_FAILURE,
|
|
79
|
+
priority: DEFAULT_ROUTING_WEIGHT_PRIORITY
|
|
80
|
+
}),
|
|
81
|
+
queue_limit: z.number().int().positive().default(DEFAULT_QUEUE_LIMIT),
|
|
82
|
+
concurrency_limit: z.number().int().positive().default(DEFAULT_GLOBAL_CONCURRENCY_LIMIT),
|
|
83
|
+
backpressure_threshold: z.number().int().nonnegative().default(DEFAULT_BACKPRESSURE_THRESHOLD),
|
|
84
|
+
circuit_breaker: CircuitBreakerConfigSchema.default({
|
|
85
|
+
failure_threshold: DEFAULT_CB_FAILURE_THRESHOLD,
|
|
86
|
+
failure_rate_threshold: DEFAULT_CB_FAILURE_RATE_THRESHOLD,
|
|
87
|
+
sliding_window_size: DEFAULT_CB_SLIDING_WINDOW_SIZE,
|
|
88
|
+
half_open_after_ms: DEFAULT_CB_HALF_OPEN_AFTER_MS,
|
|
89
|
+
half_open_max_probes: DEFAULT_CB_HALF_OPEN_MAX_PROBES,
|
|
90
|
+
success_threshold: DEFAULT_CB_SUCCESS_THRESHOLD
|
|
91
|
+
}),
|
|
92
|
+
max_expected_latency_ms: z.number().positive().default(DEFAULT_MAX_EXPECTED_LATENCY_MS)
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// src/config/loader.ts
|
|
96
|
+
var ConfigValidationError = class extends Error {
|
|
97
|
+
diagnostics;
|
|
98
|
+
constructor(diagnostics) {
|
|
99
|
+
const summary = diagnostics.map((d) => ` ${d.path}: ${d.message}`).join("\n");
|
|
100
|
+
super(`Invalid O-Switcher configuration:
|
|
101
|
+
${summary}`);
|
|
102
|
+
this.name = "ConfigValidationError";
|
|
103
|
+
this.diagnostics = diagnostics;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
var validateConfig = (raw) => {
|
|
107
|
+
const result = SwitcherConfigSchema.safeParse(raw);
|
|
108
|
+
if (result.success) {
|
|
109
|
+
const data = { ...result.data };
|
|
110
|
+
if (data.retry !== void 0 && data.retry_budget === DEFAULT_RETRY_BUDGET) {
|
|
111
|
+
data.retry_budget = data.retry;
|
|
112
|
+
}
|
|
113
|
+
if (data.timeout !== void 0 && data.max_expected_latency_ms === DEFAULT_MAX_EXPECTED_LATENCY_MS) {
|
|
114
|
+
data.max_expected_latency_ms = data.timeout;
|
|
115
|
+
}
|
|
116
|
+
if (data.targets !== void 0) {
|
|
117
|
+
const seen = /* @__PURE__ */ new Set();
|
|
118
|
+
const duplicateDiagnostics = [];
|
|
119
|
+
for (const target of data.targets) {
|
|
120
|
+
const key = `${target.provider_id}::${target.profile ?? "__default__"}`;
|
|
121
|
+
if (seen.has(key)) {
|
|
122
|
+
duplicateDiagnostics.push({
|
|
123
|
+
path: "targets",
|
|
124
|
+
message: `Duplicate provider_id + profile combination: ${key}`,
|
|
125
|
+
received: key
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
seen.add(key);
|
|
129
|
+
}
|
|
130
|
+
if (duplicateDiagnostics.length > 0) {
|
|
131
|
+
throw new ConfigValidationError(duplicateDiagnostics);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return Object.freeze(data);
|
|
135
|
+
}
|
|
136
|
+
const diagnostics = result.error.issues.map(
|
|
137
|
+
(issue) => ({
|
|
138
|
+
path: issue.path.join("."),
|
|
139
|
+
message: issue.message,
|
|
140
|
+
received: "expected" in issue ? issue.expected : void 0
|
|
141
|
+
})
|
|
142
|
+
);
|
|
143
|
+
throw new ConfigValidationError(diagnostics);
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
// src/config/discovery.ts
|
|
147
|
+
var discoverTargets = (providerConfig) => {
|
|
148
|
+
const targets = [];
|
|
149
|
+
for (const [key, value] of Object.entries(providerConfig)) {
|
|
150
|
+
if (!value) continue;
|
|
151
|
+
const raw = {
|
|
152
|
+
target_id: key,
|
|
153
|
+
provider_id: value.id ?? key,
|
|
154
|
+
capabilities: ["chat"],
|
|
155
|
+
enabled: true,
|
|
156
|
+
operator_priority: 0,
|
|
157
|
+
policy_tags: []
|
|
158
|
+
};
|
|
159
|
+
const parsed = TargetConfigSchema.parse(raw);
|
|
160
|
+
targets.push(parsed);
|
|
161
|
+
}
|
|
162
|
+
return targets;
|
|
163
|
+
};
|
|
164
|
+
var discoverTargetsFromProfiles = (store) => {
|
|
165
|
+
const targets = [];
|
|
166
|
+
for (const entry of Object.values(store)) {
|
|
167
|
+
if (!entry) continue;
|
|
168
|
+
const raw = {
|
|
169
|
+
target_id: entry.id,
|
|
170
|
+
provider_id: entry.provider,
|
|
171
|
+
profile: entry.id,
|
|
172
|
+
capabilities: ["chat"],
|
|
173
|
+
enabled: true,
|
|
174
|
+
operator_priority: 0,
|
|
175
|
+
policy_tags: []
|
|
176
|
+
};
|
|
177
|
+
const parsed = TargetConfigSchema.parse(raw);
|
|
178
|
+
targets.push(parsed);
|
|
179
|
+
}
|
|
180
|
+
return targets;
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
// src/registry/health.ts
|
|
184
|
+
var INITIAL_HEALTH_SCORE = 1;
|
|
185
|
+
var DEFAULT_ALPHA = 0.1;
|
|
186
|
+
var updateHealthScore = (currentScore, observation, alpha = DEFAULT_ALPHA) => alpha * observation + (1 - alpha) * currentScore;
|
|
187
|
+
var updateLatencyEma = (currentEma, observedMs, alpha = DEFAULT_ALPHA) => alpha * observedMs + (1 - alpha) * currentEma;
|
|
188
|
+
|
|
189
|
+
// src/registry/registry.ts
|
|
190
|
+
var TargetRegistry = class {
|
|
191
|
+
targets;
|
|
192
|
+
constructor(config) {
|
|
193
|
+
this.targets = new Map(
|
|
194
|
+
(config.targets ?? []).map((t) => [
|
|
195
|
+
t.target_id,
|
|
196
|
+
{
|
|
197
|
+
target_id: t.target_id,
|
|
198
|
+
provider_id: t.provider_id,
|
|
199
|
+
profile: t.profile,
|
|
200
|
+
endpoint_id: t.endpoint_id,
|
|
201
|
+
capabilities: [...t.capabilities],
|
|
202
|
+
enabled: t.enabled,
|
|
203
|
+
state: "Active",
|
|
204
|
+
health_score: INITIAL_HEALTH_SCORE,
|
|
205
|
+
cooldown_until: null,
|
|
206
|
+
latency_ema_ms: 0,
|
|
207
|
+
failure_score: 0,
|
|
208
|
+
operator_priority: t.operator_priority,
|
|
209
|
+
policy_tags: [...t.policy_tags]
|
|
210
|
+
}
|
|
211
|
+
])
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
/** Returns the target entry for the given id, or undefined if not found. */
|
|
215
|
+
getTarget(id) {
|
|
216
|
+
return this.targets.get(id);
|
|
217
|
+
}
|
|
218
|
+
/** Returns a readonly array of all target entries. */
|
|
219
|
+
getAllTargets() {
|
|
220
|
+
return [...this.targets.values()];
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Updates the state of a target.
|
|
224
|
+
* @returns true if the target was found and updated, false otherwise.
|
|
225
|
+
*/
|
|
226
|
+
updateState(id, newState) {
|
|
227
|
+
const target = this.targets.get(id);
|
|
228
|
+
if (!target) {
|
|
229
|
+
return false;
|
|
230
|
+
}
|
|
231
|
+
this.targets.set(id, { ...target, state: newState });
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Records a success (1) or failure (0) observation for a target.
|
|
236
|
+
* Updates health_score via EMA.
|
|
237
|
+
*/
|
|
238
|
+
recordObservation(id, observation) {
|
|
239
|
+
const target = this.targets.get(id);
|
|
240
|
+
if (!target) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
const newHealthScore = updateHealthScore(
|
|
244
|
+
target.health_score,
|
|
245
|
+
observation
|
|
246
|
+
);
|
|
247
|
+
this.targets.set(id, { ...target, health_score: newHealthScore });
|
|
248
|
+
}
|
|
249
|
+
/** Sets the cooldown_until timestamp for a target. */
|
|
250
|
+
setCooldown(id, untilMs) {
|
|
251
|
+
const target = this.targets.get(id);
|
|
252
|
+
if (!target) {
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
this.targets.set(id, { ...target, cooldown_until: untilMs });
|
|
256
|
+
}
|
|
257
|
+
/** Updates the latency EMA for a target. */
|
|
258
|
+
updateLatency(id, ms) {
|
|
259
|
+
const target = this.targets.get(id);
|
|
260
|
+
if (!target) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
const newLatency = updateLatencyEma(target.latency_ema_ms, ms);
|
|
264
|
+
this.targets.set(id, { ...target, latency_ema_ms: newLatency });
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Adds a new target entry to the registry.
|
|
268
|
+
* @returns true if added, false if target_id already exists.
|
|
269
|
+
*/
|
|
270
|
+
addTarget(entry) {
|
|
271
|
+
if (this.targets.has(entry.target_id)) {
|
|
272
|
+
return false;
|
|
273
|
+
}
|
|
274
|
+
this.targets.set(entry.target_id, entry);
|
|
275
|
+
return true;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Removes a target entry from the registry.
|
|
279
|
+
* @returns true if found and removed, false if not found.
|
|
280
|
+
*/
|
|
281
|
+
removeTarget(id) {
|
|
282
|
+
return this.targets.delete(id);
|
|
283
|
+
}
|
|
284
|
+
/** Returns a deep-frozen snapshot of all targets. */
|
|
285
|
+
getSnapshot() {
|
|
286
|
+
const snapshot = {
|
|
287
|
+
targets: [...this.targets.values()].map((t) => ({ ...t }))
|
|
288
|
+
};
|
|
289
|
+
return Object.freeze(snapshot);
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
var createRegistry = (config) => new TargetRegistry(config);
|
|
293
|
+
|
|
294
|
+
// src/registry/types.ts
|
|
295
|
+
var TARGET_STATES = [
|
|
296
|
+
"Active",
|
|
297
|
+
"CoolingDown",
|
|
298
|
+
"ReauthRequired",
|
|
299
|
+
"PolicyBlocked",
|
|
300
|
+
"CircuitOpen",
|
|
301
|
+
"CircuitHalfOpen",
|
|
302
|
+
"Draining",
|
|
303
|
+
"Disabled"
|
|
304
|
+
];
|
|
305
|
+
|
|
306
|
+
// src/audit/logger.ts
|
|
307
|
+
import pino from "pino";
|
|
308
|
+
import crypto from "crypto";
|
|
309
|
+
var REDACT_PATHS = [
|
|
310
|
+
"api_key",
|
|
311
|
+
"token",
|
|
312
|
+
"secret",
|
|
313
|
+
"password",
|
|
314
|
+
"authorization",
|
|
315
|
+
"credential",
|
|
316
|
+
"credentials",
|
|
317
|
+
"*.api_key",
|
|
318
|
+
"*.token",
|
|
319
|
+
"*.secret",
|
|
320
|
+
"*.password",
|
|
321
|
+
"*.authorization",
|
|
322
|
+
"*.credential",
|
|
323
|
+
"*.credentials"
|
|
324
|
+
];
|
|
325
|
+
var createAuditLogger = (options) => {
|
|
326
|
+
const opts = {
|
|
327
|
+
level: options?.level ?? "info",
|
|
328
|
+
redact: {
|
|
329
|
+
paths: [...REDACT_PATHS],
|
|
330
|
+
censor: "[Redacted]"
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
if (options?.destination) {
|
|
334
|
+
return pino(opts, options.destination);
|
|
335
|
+
}
|
|
336
|
+
return pino(opts);
|
|
337
|
+
};
|
|
338
|
+
var createRequestLogger = (baseLogger, requestId) => baseLogger.child({ request_id: requestId });
|
|
339
|
+
var generateCorrelationId = () => crypto.randomUUID();
|
|
340
|
+
|
|
341
|
+
// src/errors/taxonomy.ts
|
|
342
|
+
import { z as z2 } from "zod";
|
|
343
|
+
var RateLimitedSchema = z2.object({
|
|
344
|
+
class: z2.literal("RateLimited"),
|
|
345
|
+
retryable: z2.literal(true),
|
|
346
|
+
retry_after_ms: z2.number().optional(),
|
|
347
|
+
provider_reason: z2.string().optional()
|
|
348
|
+
});
|
|
349
|
+
var QuotaExhaustedSchema = z2.object({
|
|
350
|
+
class: z2.literal("QuotaExhausted"),
|
|
351
|
+
retryable: z2.literal(false),
|
|
352
|
+
provider_reason: z2.string().optional()
|
|
353
|
+
});
|
|
354
|
+
var AuthFailureSchema = z2.object({
|
|
355
|
+
class: z2.literal("AuthFailure"),
|
|
356
|
+
retryable: z2.literal(false),
|
|
357
|
+
recovery_attempted: z2.boolean().default(false)
|
|
358
|
+
});
|
|
359
|
+
var PermissionFailureSchema = z2.object({
|
|
360
|
+
class: z2.literal("PermissionFailure"),
|
|
361
|
+
retryable: z2.literal(false)
|
|
362
|
+
});
|
|
363
|
+
var PolicyFailureSchema = z2.object({
|
|
364
|
+
class: z2.literal("PolicyFailure"),
|
|
365
|
+
retryable: z2.literal(false)
|
|
366
|
+
});
|
|
367
|
+
var RegionRestrictionSchema = z2.object({
|
|
368
|
+
class: z2.literal("RegionRestriction"),
|
|
369
|
+
retryable: z2.literal(false)
|
|
370
|
+
});
|
|
371
|
+
var ModelUnavailableSchema = z2.object({
|
|
372
|
+
class: z2.literal("ModelUnavailable"),
|
|
373
|
+
retryable: z2.literal(false),
|
|
374
|
+
failover_eligible: z2.literal(true)
|
|
375
|
+
});
|
|
376
|
+
var TransientServerFailureSchema = z2.object({
|
|
377
|
+
class: z2.literal("TransientServerFailure"),
|
|
378
|
+
retryable: z2.literal(true),
|
|
379
|
+
http_status: z2.number().optional()
|
|
380
|
+
});
|
|
381
|
+
var TransportFailureSchema = z2.object({
|
|
382
|
+
class: z2.literal("TransportFailure"),
|
|
383
|
+
retryable: z2.literal(true)
|
|
384
|
+
});
|
|
385
|
+
var InterruptedExecutionSchema = z2.object({
|
|
386
|
+
class: z2.literal("InterruptedExecution"),
|
|
387
|
+
retryable: z2.literal(true),
|
|
388
|
+
partial_output_bytes: z2.number().optional()
|
|
389
|
+
});
|
|
390
|
+
var ErrorClassSchema = z2.discriminatedUnion("class", [
|
|
391
|
+
RateLimitedSchema,
|
|
392
|
+
QuotaExhaustedSchema,
|
|
393
|
+
AuthFailureSchema,
|
|
394
|
+
PermissionFailureSchema,
|
|
395
|
+
PolicyFailureSchema,
|
|
396
|
+
RegionRestrictionSchema,
|
|
397
|
+
ModelUnavailableSchema,
|
|
398
|
+
TransientServerFailureSchema,
|
|
399
|
+
TransportFailureSchema,
|
|
400
|
+
InterruptedExecutionSchema
|
|
401
|
+
]);
|
|
402
|
+
var isRetryable = (errorClass) => errorClass.retryable;
|
|
403
|
+
var getTargetStateTransition = (errorClass) => {
|
|
404
|
+
switch (errorClass.class) {
|
|
405
|
+
case "RateLimited":
|
|
406
|
+
return "CoolingDown";
|
|
407
|
+
case "QuotaExhausted":
|
|
408
|
+
return "Disabled";
|
|
409
|
+
case "AuthFailure":
|
|
410
|
+
return "ReauthRequired";
|
|
411
|
+
case "PermissionFailure":
|
|
412
|
+
return "PolicyBlocked";
|
|
413
|
+
case "PolicyFailure":
|
|
414
|
+
return "PolicyBlocked";
|
|
415
|
+
case "RegionRestriction":
|
|
416
|
+
return "Disabled";
|
|
417
|
+
case "ModelUnavailable":
|
|
418
|
+
return null;
|
|
419
|
+
case "TransientServerFailure":
|
|
420
|
+
return null;
|
|
421
|
+
case "TransportFailure":
|
|
422
|
+
return null;
|
|
423
|
+
case "InterruptedExecution":
|
|
424
|
+
return null;
|
|
425
|
+
}
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
// src/retry/backoff.ts
|
|
429
|
+
var DEFAULT_BACKOFF_PARAMS = {
|
|
430
|
+
base_ms: 1e3,
|
|
431
|
+
multiplier: 2,
|
|
432
|
+
max_ms: 3e4,
|
|
433
|
+
jitter: "full"
|
|
434
|
+
};
|
|
435
|
+
var computeBackoffMs = (attempt, params, retryAfterMs) => {
|
|
436
|
+
const rawDelay = Math.min(
|
|
437
|
+
params.base_ms * Math.pow(params.multiplier, attempt),
|
|
438
|
+
params.max_ms
|
|
439
|
+
);
|
|
440
|
+
const jitteredDelay = params.jitter === "full" ? Math.random() * rawDelay : params.jitter === "equal" ? rawDelay / 2 + Math.random() * (rawDelay / 2) : rawDelay;
|
|
441
|
+
return Math.max(jitteredDelay, retryAfterMs ?? 0);
|
|
442
|
+
};
|
|
443
|
+
|
|
444
|
+
// src/retry/retry-policy.ts
|
|
445
|
+
var createRetryPolicy = (budget, backoffParams = DEFAULT_BACKOFF_PARAMS) => ({
|
|
446
|
+
budget,
|
|
447
|
+
backoffParams,
|
|
448
|
+
decide: (classification, attemptsSoFar) => {
|
|
449
|
+
const errorClass = classification.error_class;
|
|
450
|
+
if (!isRetryable(errorClass)) {
|
|
451
|
+
return {
|
|
452
|
+
action: "abort",
|
|
453
|
+
reason: `Non-retryable: ${errorClass.class}`,
|
|
454
|
+
target_state: getTargetStateTransition(errorClass)
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
if (attemptsSoFar >= budget) {
|
|
458
|
+
return {
|
|
459
|
+
action: "exhausted",
|
|
460
|
+
attempts_used: attemptsSoFar
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
const retryAfterMs = errorClass.class === "RateLimited" ? errorClass.retry_after_ms : void 0;
|
|
464
|
+
const delayMs = computeBackoffMs(attemptsSoFar, backoffParams, retryAfterMs);
|
|
465
|
+
return {
|
|
466
|
+
action: "retry",
|
|
467
|
+
delay_ms: delayMs,
|
|
468
|
+
attempt: attemptsSoFar + 1
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
// src/routing/types.ts
|
|
474
|
+
var EXCLUSION_REASONS = [
|
|
475
|
+
"disabled",
|
|
476
|
+
"policy_blocked",
|
|
477
|
+
"reauth_required",
|
|
478
|
+
"cooldown_active",
|
|
479
|
+
"circuit_open",
|
|
480
|
+
"capability_mismatch",
|
|
481
|
+
"draining"
|
|
482
|
+
];
|
|
483
|
+
var ADMISSION_RESULTS = [
|
|
484
|
+
"admitted",
|
|
485
|
+
"queued",
|
|
486
|
+
"degraded",
|
|
487
|
+
"rejected"
|
|
488
|
+
];
|
|
489
|
+
|
|
490
|
+
// src/routing/events.ts
|
|
491
|
+
import { EventEmitter } from "eventemitter3";
|
|
492
|
+
var createRoutingEventBus = () => new EventEmitter();
|
|
493
|
+
|
|
494
|
+
// src/routing/dual-breaker.ts
|
|
495
|
+
import {
|
|
496
|
+
ConsecutiveBreaker,
|
|
497
|
+
CountBreaker
|
|
498
|
+
} from "cockatiel";
|
|
499
|
+
var DualBreaker = class {
|
|
500
|
+
consecutive;
|
|
501
|
+
count;
|
|
502
|
+
constructor(consecutiveThreshold, countOpts) {
|
|
503
|
+
this.consecutive = new ConsecutiveBreaker(consecutiveThreshold);
|
|
504
|
+
this.count = new CountBreaker(countOpts);
|
|
505
|
+
}
|
|
506
|
+
/** Serializable state for both internal breakers. */
|
|
507
|
+
get state() {
|
|
508
|
+
return {
|
|
509
|
+
consecutive: this.consecutive.state,
|
|
510
|
+
count: this.count.state
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
/** Restore state for both internal breakers. */
|
|
514
|
+
set state(value) {
|
|
515
|
+
const v = value;
|
|
516
|
+
this.consecutive.state = v.consecutive;
|
|
517
|
+
this.count.state = v.count;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Record success on both breakers.
|
|
521
|
+
* ConsecutiveBreaker.success() takes no args (resets counter).
|
|
522
|
+
* CountBreaker.success() takes CircuitState.
|
|
523
|
+
*/
|
|
524
|
+
success(state) {
|
|
525
|
+
this.consecutive.success();
|
|
526
|
+
this.count.success(state);
|
|
527
|
+
}
|
|
528
|
+
/**
|
|
529
|
+
* Record failure on both breakers.
|
|
530
|
+
* Returns true if EITHER breaker trips (OR logic).
|
|
531
|
+
* ConsecutiveBreaker.failure() takes no args.
|
|
532
|
+
* CountBreaker.failure() takes CircuitState.
|
|
533
|
+
*/
|
|
534
|
+
failure(state) {
|
|
535
|
+
const consecutiveTripped = this.consecutive.failure();
|
|
536
|
+
const countTripped = this.count.failure(state);
|
|
537
|
+
return consecutiveTripped || countTripped;
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
|
|
541
|
+
// src/routing/circuit-breaker.ts
|
|
542
|
+
import {
|
|
543
|
+
CircuitState as CircuitState2,
|
|
544
|
+
handleAll,
|
|
545
|
+
circuitBreaker,
|
|
546
|
+
BrokenCircuitError,
|
|
547
|
+
IsolatedCircuitError
|
|
548
|
+
} from "cockatiel";
|
|
549
|
+
import "eventemitter3";
|
|
550
|
+
var COCKATIEL_TO_TARGET = /* @__PURE__ */ new Map([
|
|
551
|
+
[CircuitState2.Closed, "Active"],
|
|
552
|
+
[CircuitState2.Open, "CircuitOpen"],
|
|
553
|
+
[CircuitState2.HalfOpen, "CircuitHalfOpen"],
|
|
554
|
+
[CircuitState2.Isolated, "Disabled"]
|
|
555
|
+
]);
|
|
556
|
+
var toTargetState = (state) => COCKATIEL_TO_TARGET.get(state) ?? "Disabled";
|
|
557
|
+
var createCircuitBreaker = (targetId, config, eventBus) => {
|
|
558
|
+
const createPolicy = () => {
|
|
559
|
+
const breaker = new DualBreaker(config.failure_threshold, {
|
|
560
|
+
threshold: config.failure_rate_threshold,
|
|
561
|
+
size: config.sliding_window_size
|
|
562
|
+
});
|
|
563
|
+
const policy = circuitBreaker(handleAll, {
|
|
564
|
+
halfOpenAfter: config.half_open_after_ms,
|
|
565
|
+
breaker
|
|
566
|
+
});
|
|
567
|
+
return { policy, breaker };
|
|
568
|
+
};
|
|
569
|
+
let current = createPolicy();
|
|
570
|
+
let previousState = CircuitState2.Closed;
|
|
571
|
+
let openedAtMs = 0;
|
|
572
|
+
const subscribeEvents = (policy) => {
|
|
573
|
+
policy.onStateChange((newState) => {
|
|
574
|
+
if (newState === CircuitState2.Open) {
|
|
575
|
+
openedAtMs = Date.now();
|
|
576
|
+
}
|
|
577
|
+
const from = toTargetState(previousState);
|
|
578
|
+
const to = toTargetState(newState);
|
|
579
|
+
previousState = newState;
|
|
580
|
+
if (from !== to && eventBus) {
|
|
581
|
+
eventBus.emit("circuit_state_change", {
|
|
582
|
+
target_id: targetId,
|
|
583
|
+
from,
|
|
584
|
+
to,
|
|
585
|
+
reason: `state_transition`
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
});
|
|
589
|
+
};
|
|
590
|
+
subscribeEvents(current.policy);
|
|
591
|
+
const effectiveState = () => {
|
|
592
|
+
const raw = current.policy.state;
|
|
593
|
+
if (raw === CircuitState2.Open && openedAtMs > 0 && Date.now() - openedAtMs >= config.half_open_after_ms) {
|
|
594
|
+
return CircuitState2.HalfOpen;
|
|
595
|
+
}
|
|
596
|
+
return raw;
|
|
597
|
+
};
|
|
598
|
+
return {
|
|
599
|
+
state() {
|
|
600
|
+
return toTargetState(effectiveState());
|
|
601
|
+
},
|
|
602
|
+
async recordSuccess() {
|
|
603
|
+
try {
|
|
604
|
+
await current.policy.execute(async () => void 0);
|
|
605
|
+
} catch (err) {
|
|
606
|
+
if (err instanceof BrokenCircuitError || err instanceof IsolatedCircuitError) {
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
throw err;
|
|
610
|
+
}
|
|
611
|
+
},
|
|
612
|
+
async recordFailure() {
|
|
613
|
+
try {
|
|
614
|
+
await current.policy.execute(async () => {
|
|
615
|
+
throw new Error("recorded-failure");
|
|
616
|
+
});
|
|
617
|
+
} catch {
|
|
618
|
+
}
|
|
619
|
+
},
|
|
620
|
+
allowRequest() {
|
|
621
|
+
return effectiveState() !== CircuitState2.Open;
|
|
622
|
+
},
|
|
623
|
+
reset() {
|
|
624
|
+
current = createPolicy();
|
|
625
|
+
previousState = CircuitState2.Closed;
|
|
626
|
+
openedAtMs = 0;
|
|
627
|
+
subscribeEvents(current.policy);
|
|
628
|
+
}
|
|
629
|
+
};
|
|
630
|
+
};
|
|
631
|
+
|
|
632
|
+
// src/routing/concurrency-tracker.ts
|
|
633
|
+
var createConcurrencyTracker = (defaultLimit) => {
|
|
634
|
+
const state = /* @__PURE__ */ new Map();
|
|
635
|
+
const getOrCreate = (target_id) => {
|
|
636
|
+
const existing = state.get(target_id);
|
|
637
|
+
if (existing) return existing;
|
|
638
|
+
const entry = { active: 0, limit: defaultLimit };
|
|
639
|
+
state.set(target_id, entry);
|
|
640
|
+
return entry;
|
|
641
|
+
};
|
|
642
|
+
return {
|
|
643
|
+
acquire(target_id) {
|
|
644
|
+
const entry = getOrCreate(target_id);
|
|
645
|
+
if (entry.active >= entry.limit) return false;
|
|
646
|
+
entry.active += 1;
|
|
647
|
+
return true;
|
|
648
|
+
},
|
|
649
|
+
release(target_id) {
|
|
650
|
+
const entry = state.get(target_id);
|
|
651
|
+
if (!entry || entry.active <= 0) return;
|
|
652
|
+
entry.active -= 1;
|
|
653
|
+
},
|
|
654
|
+
headroom(target_id) {
|
|
655
|
+
const entry = getOrCreate(target_id);
|
|
656
|
+
return Math.max(0, entry.limit - entry.active);
|
|
657
|
+
},
|
|
658
|
+
active(target_id) {
|
|
659
|
+
const entry = state.get(target_id);
|
|
660
|
+
return entry ? entry.active : 0;
|
|
661
|
+
},
|
|
662
|
+
setLimit(target_id, limit) {
|
|
663
|
+
const entry = getOrCreate(target_id);
|
|
664
|
+
entry.limit = limit;
|
|
665
|
+
}
|
|
666
|
+
};
|
|
667
|
+
};
|
|
668
|
+
|
|
669
|
+
// src/routing/policy-engine.ts
|
|
670
|
+
var normalizeLatency = (latencyEmaMs, maxExpectedMs) => Math.min(latencyEmaMs / maxExpectedMs, 1);
|
|
671
|
+
var computeScore = (target, weights, maxLatencyMs) => weights.health * target.health_score - weights.latency * normalizeLatency(target.latency_ema_ms, maxLatencyMs) - weights.failure * target.failure_score + weights.priority * target.operator_priority;
|
|
672
|
+
var getExclusionReason = (target, circuitBreakers, requiredCapabilities, excludeTargets, nowMs) => {
|
|
673
|
+
if (!target.enabled) {
|
|
674
|
+
return "disabled";
|
|
675
|
+
}
|
|
676
|
+
if (target.state === "PolicyBlocked") {
|
|
677
|
+
return "policy_blocked";
|
|
678
|
+
}
|
|
679
|
+
if (target.state === "ReauthRequired") {
|
|
680
|
+
return "reauth_required";
|
|
681
|
+
}
|
|
682
|
+
if (target.state === "Draining") {
|
|
683
|
+
return "draining";
|
|
684
|
+
}
|
|
685
|
+
if (target.state === "Disabled") {
|
|
686
|
+
return "disabled";
|
|
687
|
+
}
|
|
688
|
+
const breaker = circuitBreakers.get(target.target_id);
|
|
689
|
+
if (breaker !== void 0 && !breaker.allowRequest()) {
|
|
690
|
+
return "circuit_open";
|
|
691
|
+
}
|
|
692
|
+
if (target.cooldown_until !== null && target.cooldown_until > nowMs) {
|
|
693
|
+
return "cooldown_active";
|
|
694
|
+
}
|
|
695
|
+
if (requiredCapabilities.length > 0 && !requiredCapabilities.every((cap) => target.capabilities.includes(cap))) {
|
|
696
|
+
return "capability_mismatch";
|
|
697
|
+
}
|
|
698
|
+
return null;
|
|
699
|
+
};
|
|
700
|
+
var selectTarget = (snapshot, circuitBreakers, requiredCapabilities, weights, maxLatencyMs, nowMs, request_id, excludeTargets, logger) => {
|
|
701
|
+
const excluded = [];
|
|
702
|
+
const eligible = [];
|
|
703
|
+
for (const target of snapshot.targets) {
|
|
704
|
+
if (excludeTargets !== void 0 && excludeTargets.has(target.target_id)) {
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
const reason = getExclusionReason(
|
|
708
|
+
target,
|
|
709
|
+
circuitBreakers,
|
|
710
|
+
requiredCapabilities,
|
|
711
|
+
excludeTargets ?? /* @__PURE__ */ new Set(),
|
|
712
|
+
nowMs
|
|
713
|
+
);
|
|
714
|
+
if (reason !== null) {
|
|
715
|
+
excluded.push({ target_id: target.target_id, reason });
|
|
716
|
+
} else {
|
|
717
|
+
eligible.push(target);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
const scored = eligible.map((target) => ({
|
|
721
|
+
target,
|
|
722
|
+
score: computeScore(target, weights, maxLatencyMs)
|
|
723
|
+
}));
|
|
724
|
+
scored.sort((a, b) => {
|
|
725
|
+
const scoreDiff = b.score - a.score;
|
|
726
|
+
if (scoreDiff !== 0) {
|
|
727
|
+
return scoreDiff;
|
|
728
|
+
}
|
|
729
|
+
return a.target.target_id.localeCompare(b.target.target_id);
|
|
730
|
+
});
|
|
731
|
+
const candidates = scored.map((s) => ({
|
|
732
|
+
target_id: s.target.target_id,
|
|
733
|
+
score: s.score,
|
|
734
|
+
health_score: s.target.health_score,
|
|
735
|
+
latency_ema_ms: s.target.latency_ema_ms
|
|
736
|
+
}));
|
|
737
|
+
const firstCandidate = candidates.length > 0 ? candidates[0] : void 0;
|
|
738
|
+
const selected = firstCandidate !== void 0 ? {
|
|
739
|
+
target_id: firstCandidate.target_id,
|
|
740
|
+
score: firstCandidate.score,
|
|
741
|
+
rank: 0
|
|
742
|
+
} : null;
|
|
743
|
+
const record = {
|
|
744
|
+
request_id,
|
|
745
|
+
timestamp_ms: nowMs,
|
|
746
|
+
candidates,
|
|
747
|
+
excluded,
|
|
748
|
+
selected
|
|
749
|
+
};
|
|
750
|
+
if (logger !== void 0) {
|
|
751
|
+
const excludedSummary = {};
|
|
752
|
+
for (const e of excluded) {
|
|
753
|
+
excludedSummary[e.reason] = (excludedSummary[e.reason] ?? 0) + 1;
|
|
754
|
+
}
|
|
755
|
+
const logFields = {
|
|
756
|
+
event: "target_selected",
|
|
757
|
+
component: "policy-engine",
|
|
758
|
+
request_id,
|
|
759
|
+
selected: selected?.target_id ?? null,
|
|
760
|
+
score: selected?.score ?? null,
|
|
761
|
+
candidates_count: candidates.length,
|
|
762
|
+
excluded_count: excluded.length,
|
|
763
|
+
excluded_summary: excludedSummary
|
|
764
|
+
};
|
|
765
|
+
if (selected !== null) {
|
|
766
|
+
logger.info(logFields, "Target selected");
|
|
767
|
+
} else {
|
|
768
|
+
logger.warn(logFields, "No eligible target");
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
return record;
|
|
772
|
+
};
|
|
773
|
+
|
|
774
|
+
// src/routing/cooldown.ts
|
|
775
|
+
var DEFAULT_COOLDOWN_BASE_MS = 5e3;
|
|
776
|
+
var computeCooldownMs = (errorClass, consecutiveFailures, backoffParams, maxMs) => {
|
|
777
|
+
const noJitterParams = {
|
|
778
|
+
...backoffParams,
|
|
779
|
+
jitter: "none"
|
|
780
|
+
};
|
|
781
|
+
let base;
|
|
782
|
+
if (errorClass.class === "RateLimited" && "retry_after_ms" in errorClass && errorClass.retry_after_ms !== void 0) {
|
|
783
|
+
base = Math.min(errorClass.retry_after_ms, maxMs);
|
|
784
|
+
} else if (errorClass.class === "RateLimited") {
|
|
785
|
+
base = computeBackoffMs(consecutiveFailures, noJitterParams);
|
|
786
|
+
} else if (errorClass.class === "TransientServerFailure") {
|
|
787
|
+
base = computeBackoffMs(consecutiveFailures, noJitterParams) / 2;
|
|
788
|
+
} else {
|
|
789
|
+
base = DEFAULT_COOLDOWN_BASE_MS;
|
|
790
|
+
}
|
|
791
|
+
base = Math.min(base, maxMs);
|
|
792
|
+
const jitterFraction = 0.1 + Math.random() * 0.15;
|
|
793
|
+
const jitter = jitterFraction * base;
|
|
794
|
+
return Math.min(base + jitter, maxMs);
|
|
795
|
+
};
|
|
796
|
+
var createCooldownManager = (registry, eventBus) => ({
|
|
797
|
+
setCooldown(target_id, durationMs, errorClass) {
|
|
798
|
+
const untilMs = Date.now() + durationMs;
|
|
799
|
+
registry.setCooldown(target_id, untilMs);
|
|
800
|
+
registry.updateState(target_id, "CoolingDown");
|
|
801
|
+
eventBus?.emit("cooldown_set", {
|
|
802
|
+
target_id,
|
|
803
|
+
until_ms: untilMs,
|
|
804
|
+
error_class: errorClass
|
|
805
|
+
});
|
|
806
|
+
},
|
|
807
|
+
checkExpired(nowMs) {
|
|
808
|
+
const expired = [];
|
|
809
|
+
for (const target of registry.getAllTargets()) {
|
|
810
|
+
if (target.state === "CoolingDown" && target.cooldown_until !== null && target.cooldown_until <= nowMs) {
|
|
811
|
+
registry.setCooldown(target.target_id, 0);
|
|
812
|
+
registry.updateState(target.target_id, "Active");
|
|
813
|
+
eventBus?.emit("cooldown_expired", { target_id: target.target_id });
|
|
814
|
+
expired.push(target.target_id);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
return expired;
|
|
818
|
+
},
|
|
819
|
+
isInCooldown(target_id, nowMs) {
|
|
820
|
+
const target = registry.getTarget(target_id);
|
|
821
|
+
if (!target) {
|
|
822
|
+
return false;
|
|
823
|
+
}
|
|
824
|
+
return target.cooldown_until !== null && target.cooldown_until > nowMs;
|
|
825
|
+
}
|
|
826
|
+
});
|
|
827
|
+
|
|
828
|
+
// src/routing/admission.ts
|
|
829
|
+
import PQueue from "p-queue";
|
|
830
|
+
var checkHardRejects = (ctx) => {
|
|
831
|
+
if (!ctx.hasEligibleTargets) {
|
|
832
|
+
return { result: "rejected", reason: "no eligible targets available" };
|
|
833
|
+
}
|
|
834
|
+
if (ctx.operatorPaused) {
|
|
835
|
+
return { result: "rejected", reason: "operator pause active" };
|
|
836
|
+
}
|
|
837
|
+
if (ctx.retryBudgetRemaining <= 0) {
|
|
838
|
+
return { result: "rejected", reason: "retry budget exhausted" };
|
|
839
|
+
}
|
|
840
|
+
if (ctx.failoverBudgetRemaining <= 0) {
|
|
841
|
+
return { result: "rejected", reason: "failover budget exhausted" };
|
|
842
|
+
}
|
|
843
|
+
return null;
|
|
844
|
+
};
|
|
845
|
+
var createAdmissionController = (config, eventBus) => {
|
|
846
|
+
const queue = new PQueue({
|
|
847
|
+
concurrency: config.concurrencyLimit
|
|
848
|
+
});
|
|
849
|
+
const emitDecision = (request_id, decision) => {
|
|
850
|
+
eventBus?.emit("admission_decision", {
|
|
851
|
+
request_id,
|
|
852
|
+
result: decision.result,
|
|
853
|
+
reason: decision.reason
|
|
854
|
+
});
|
|
855
|
+
};
|
|
856
|
+
const controller = {
|
|
857
|
+
async admit(request_id, ctx) {
|
|
858
|
+
const hardReject = checkHardRejects(ctx);
|
|
859
|
+
if (hardReject) {
|
|
860
|
+
emitDecision(request_id, hardReject);
|
|
861
|
+
return hardReject;
|
|
862
|
+
}
|
|
863
|
+
if (queue.size > config.backpressureThreshold) {
|
|
864
|
+
const decision2 = {
|
|
865
|
+
result: "degraded",
|
|
866
|
+
reason: `backpressure: ${queue.size} queued > ${config.backpressureThreshold} threshold`
|
|
867
|
+
};
|
|
868
|
+
emitDecision(request_id, decision2);
|
|
869
|
+
return decision2;
|
|
870
|
+
}
|
|
871
|
+
const total = queue.size + queue.pending;
|
|
872
|
+
if (total >= config.queueLimit + config.concurrencyLimit) {
|
|
873
|
+
const decision2 = {
|
|
874
|
+
result: "rejected",
|
|
875
|
+
reason: `queue full: ${total} >= ${config.queueLimit + config.concurrencyLimit}`
|
|
876
|
+
};
|
|
877
|
+
emitDecision(request_id, decision2);
|
|
878
|
+
return decision2;
|
|
879
|
+
}
|
|
880
|
+
const decision = {
|
|
881
|
+
result: "admitted",
|
|
882
|
+
reason: "capacity available"
|
|
883
|
+
};
|
|
884
|
+
emitDecision(request_id, decision);
|
|
885
|
+
return decision;
|
|
886
|
+
},
|
|
887
|
+
async execute(fn, request_id, ctx) {
|
|
888
|
+
const decision = await controller.admit(request_id, ctx);
|
|
889
|
+
if (decision.result === "rejected") {
|
|
890
|
+
return { decision };
|
|
891
|
+
}
|
|
892
|
+
const result = await queue.add(fn);
|
|
893
|
+
return { decision, result };
|
|
894
|
+
},
|
|
895
|
+
get pending() {
|
|
896
|
+
return queue.pending;
|
|
897
|
+
},
|
|
898
|
+
get size() {
|
|
899
|
+
return queue.size;
|
|
900
|
+
}
|
|
901
|
+
};
|
|
902
|
+
return controller;
|
|
903
|
+
};
|
|
904
|
+
|
|
905
|
+
// src/routing/log-subscriber.ts
|
|
906
|
+
var createLogSubscriber = (eventBus, logger) => {
|
|
907
|
+
const log = logger.child({ component: "routing" });
|
|
908
|
+
const onCircuitStateChange = (payload) => {
|
|
909
|
+
log.info(
|
|
910
|
+
{
|
|
911
|
+
event: "circuit_state_change",
|
|
912
|
+
target_id: payload.target_id,
|
|
913
|
+
from: payload.from,
|
|
914
|
+
to: payload.to,
|
|
915
|
+
reason: payload.reason
|
|
916
|
+
},
|
|
917
|
+
"Circuit breaker transition"
|
|
918
|
+
);
|
|
919
|
+
};
|
|
920
|
+
const onCooldownSet = (payload) => {
|
|
921
|
+
log.info(
|
|
922
|
+
{
|
|
923
|
+
event: "cooldown_set",
|
|
924
|
+
target_id: payload.target_id,
|
|
925
|
+
duration_ms: payload.until_ms - Date.now(),
|
|
926
|
+
error_class: payload.error_class
|
|
927
|
+
},
|
|
928
|
+
"Cooldown set"
|
|
929
|
+
);
|
|
930
|
+
};
|
|
931
|
+
const onCooldownExpired = (payload) => {
|
|
932
|
+
log.info(
|
|
933
|
+
{
|
|
934
|
+
event: "cooldown_expired",
|
|
935
|
+
target_id: payload.target_id
|
|
936
|
+
},
|
|
937
|
+
"Cooldown expired"
|
|
938
|
+
);
|
|
939
|
+
};
|
|
940
|
+
const onAdmissionDecision = (payload) => {
|
|
941
|
+
const fields = {
|
|
942
|
+
event: "admission_decision",
|
|
943
|
+
request_id: payload.request_id,
|
|
944
|
+
result: payload.result,
|
|
945
|
+
reason: payload.reason
|
|
946
|
+
};
|
|
947
|
+
if (payload.result === "admitted" || payload.result === "queued") {
|
|
948
|
+
log.info(fields, "Admission decision");
|
|
949
|
+
} else {
|
|
950
|
+
log.warn(fields, "Admission decision");
|
|
951
|
+
}
|
|
952
|
+
};
|
|
953
|
+
const onHealthUpdated = (payload) => {
|
|
954
|
+
log.debug(
|
|
955
|
+
{
|
|
956
|
+
event: "health_updated",
|
|
957
|
+
target_id: payload.target_id,
|
|
958
|
+
old_score: payload.old_score,
|
|
959
|
+
new_score: payload.new_score
|
|
960
|
+
},
|
|
961
|
+
"Health score updated"
|
|
962
|
+
);
|
|
963
|
+
};
|
|
964
|
+
const onTargetExcluded = (payload) => {
|
|
965
|
+
log.debug(
|
|
966
|
+
{
|
|
967
|
+
event: "target_excluded",
|
|
968
|
+
target_id: payload.target_id,
|
|
969
|
+
reason: payload.reason
|
|
970
|
+
},
|
|
971
|
+
"Target excluded"
|
|
972
|
+
);
|
|
973
|
+
};
|
|
974
|
+
eventBus.on("circuit_state_change", onCircuitStateChange);
|
|
975
|
+
eventBus.on("cooldown_set", onCooldownSet);
|
|
976
|
+
eventBus.on("cooldown_expired", onCooldownExpired);
|
|
977
|
+
eventBus.on("admission_decision", onAdmissionDecision);
|
|
978
|
+
eventBus.on("health_updated", onHealthUpdated);
|
|
979
|
+
eventBus.on("target_excluded", onTargetExcluded);
|
|
980
|
+
return () => {
|
|
981
|
+
eventBus.off("circuit_state_change", onCircuitStateChange);
|
|
982
|
+
eventBus.off("cooldown_set", onCooldownSet);
|
|
983
|
+
eventBus.off("cooldown_expired", onCooldownExpired);
|
|
984
|
+
eventBus.off("admission_decision", onAdmissionDecision);
|
|
985
|
+
eventBus.off("health_updated", onHealthUpdated);
|
|
986
|
+
eventBus.off("target_excluded", onTargetExcluded);
|
|
987
|
+
};
|
|
988
|
+
};
|
|
989
|
+
|
|
990
|
+
// src/routing/failover.ts
|
|
991
|
+
var createFailoverOrchestrator = (deps) => ({
|
|
992
|
+
async execute(request_id, attemptFn, requiredCapabilities) {
|
|
993
|
+
const log = deps.logger?.child({ component: "failover", request_id });
|
|
994
|
+
const attempts = [];
|
|
995
|
+
const failedTargets = /* @__PURE__ */ new Set();
|
|
996
|
+
let totalRetries = 0;
|
|
997
|
+
let totalFailovers = 0;
|
|
998
|
+
let lastErrorClass;
|
|
999
|
+
log?.info(
|
|
1000
|
+
{ event: "failover_start", retry_budget: deps.retryBudget, failover_budget: deps.failoverBudget },
|
|
1001
|
+
"Failover loop started"
|
|
1002
|
+
);
|
|
1003
|
+
for (let failoverNo = 0; failoverNo <= deps.failoverBudget; failoverNo++) {
|
|
1004
|
+
deps.cooldownManager.checkExpired(Date.now());
|
|
1005
|
+
const snapshot = deps.registry.getSnapshot();
|
|
1006
|
+
const selection = selectTarget(
|
|
1007
|
+
snapshot,
|
|
1008
|
+
deps.circuitBreakers,
|
|
1009
|
+
requiredCapabilities,
|
|
1010
|
+
deps.weights,
|
|
1011
|
+
deps.maxLatencyMs,
|
|
1012
|
+
Date.now(),
|
|
1013
|
+
request_id,
|
|
1014
|
+
failedTargets
|
|
1015
|
+
);
|
|
1016
|
+
if (!selection.selected) {
|
|
1017
|
+
log?.warn(
|
|
1018
|
+
{ event: "no_eligible_target", failover_no: failoverNo },
|
|
1019
|
+
"No eligible target available"
|
|
1020
|
+
);
|
|
1021
|
+
return {
|
|
1022
|
+
outcome: "failure",
|
|
1023
|
+
reason: "no eligible target available",
|
|
1024
|
+
attempts,
|
|
1025
|
+
total_retries: totalRetries,
|
|
1026
|
+
total_failovers: totalFailovers,
|
|
1027
|
+
last_error_class: lastErrorClass
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
const targetId = selection.selected.target_id;
|
|
1031
|
+
const retryPolicy = createRetryPolicy(
|
|
1032
|
+
deps.retryBudget,
|
|
1033
|
+
deps.backoffParams
|
|
1034
|
+
);
|
|
1035
|
+
for (let retry = 0; retry <= deps.retryBudget; retry++) {
|
|
1036
|
+
const attemptNo = retry + 1;
|
|
1037
|
+
const acquired = deps.concurrency.acquire(targetId);
|
|
1038
|
+
if (!acquired) {
|
|
1039
|
+
log?.warn(
|
|
1040
|
+
{ event: "concurrency_full", target_id: targetId, failover_no: failoverNo },
|
|
1041
|
+
"No concurrency headroom \u2014 failing over"
|
|
1042
|
+
);
|
|
1043
|
+
failedTargets.add(targetId);
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
let attemptResult;
|
|
1047
|
+
try {
|
|
1048
|
+
attemptResult = await attemptFn(targetId);
|
|
1049
|
+
} finally {
|
|
1050
|
+
deps.concurrency.release(targetId);
|
|
1051
|
+
}
|
|
1052
|
+
if (attemptResult.success) {
|
|
1053
|
+
const cb2 = deps.circuitBreakers.get(targetId);
|
|
1054
|
+
if (cb2) {
|
|
1055
|
+
await cb2.recordSuccess();
|
|
1056
|
+
}
|
|
1057
|
+
deps.registry.recordObservation(targetId, 1);
|
|
1058
|
+
deps.registry.updateLatency(targetId, attemptResult.latency_ms);
|
|
1059
|
+
log?.info(
|
|
1060
|
+
{ event: "attempt_success", target_id: targetId, attempt_no: attemptNo, failover_no: failoverNo, latency_ms: attemptResult.latency_ms },
|
|
1061
|
+
"Attempt succeeded"
|
|
1062
|
+
);
|
|
1063
|
+
attempts.push({
|
|
1064
|
+
target_id: targetId,
|
|
1065
|
+
attempt_no: attemptNo,
|
|
1066
|
+
failover_no: failoverNo,
|
|
1067
|
+
outcome: "success",
|
|
1068
|
+
latency_ms: attemptResult.latency_ms,
|
|
1069
|
+
selection
|
|
1070
|
+
});
|
|
1071
|
+
return {
|
|
1072
|
+
outcome: "success",
|
|
1073
|
+
target_id: targetId,
|
|
1074
|
+
attempts,
|
|
1075
|
+
total_retries: totalRetries,
|
|
1076
|
+
total_failovers: totalFailovers,
|
|
1077
|
+
value: attemptResult.value
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
const errorClass = attemptResult.error_class;
|
|
1081
|
+
lastErrorClass = errorClass.class;
|
|
1082
|
+
const cb = deps.circuitBreakers.get(targetId);
|
|
1083
|
+
if (cb) {
|
|
1084
|
+
await cb.recordFailure();
|
|
1085
|
+
}
|
|
1086
|
+
deps.registry.recordObservation(targetId, 0);
|
|
1087
|
+
const stateTransition = getTargetStateTransition(errorClass);
|
|
1088
|
+
if (stateTransition) {
|
|
1089
|
+
deps.registry.updateState(targetId, stateTransition);
|
|
1090
|
+
}
|
|
1091
|
+
if (errorClass.class === "RateLimited") {
|
|
1092
|
+
const cooldownMs = computeCooldownMs(
|
|
1093
|
+
errorClass,
|
|
1094
|
+
retry,
|
|
1095
|
+
deps.backoffParams,
|
|
1096
|
+
deps.backoffParams.max_ms
|
|
1097
|
+
);
|
|
1098
|
+
deps.cooldownManager.setCooldown(
|
|
1099
|
+
targetId,
|
|
1100
|
+
cooldownMs,
|
|
1101
|
+
errorClass.class
|
|
1102
|
+
);
|
|
1103
|
+
}
|
|
1104
|
+
if (errorClass.class === "ModelUnavailable") {
|
|
1105
|
+
log?.info(
|
|
1106
|
+
{ event: "early_failover", target_id: targetId, error_class: errorClass.class, failover_no: failoverNo },
|
|
1107
|
+
"Early failover \u2014 ModelUnavailable"
|
|
1108
|
+
);
|
|
1109
|
+
attempts.push({
|
|
1110
|
+
target_id: targetId,
|
|
1111
|
+
attempt_no: attemptNo,
|
|
1112
|
+
failover_no: failoverNo,
|
|
1113
|
+
outcome: "failover",
|
|
1114
|
+
error_class: errorClass.class,
|
|
1115
|
+
latency_ms: attemptResult.latency_ms,
|
|
1116
|
+
selection
|
|
1117
|
+
});
|
|
1118
|
+
failedTargets.add(targetId);
|
|
1119
|
+
totalFailovers++;
|
|
1120
|
+
break;
|
|
1121
|
+
}
|
|
1122
|
+
if (!isRetryable(errorClass)) {
|
|
1123
|
+
log?.warn(
|
|
1124
|
+
{ event: "abort", target_id: targetId, error_class: errorClass.class },
|
|
1125
|
+
"Non-retryable error \u2014 aborting"
|
|
1126
|
+
);
|
|
1127
|
+
attempts.push({
|
|
1128
|
+
target_id: targetId,
|
|
1129
|
+
attempt_no: attemptNo,
|
|
1130
|
+
failover_no: failoverNo,
|
|
1131
|
+
outcome: "abort",
|
|
1132
|
+
error_class: errorClass.class,
|
|
1133
|
+
latency_ms: attemptResult.latency_ms,
|
|
1134
|
+
selection
|
|
1135
|
+
});
|
|
1136
|
+
return {
|
|
1137
|
+
outcome: "failure",
|
|
1138
|
+
reason: `non-retryable: ${errorClass.class}`,
|
|
1139
|
+
attempts,
|
|
1140
|
+
total_retries: totalRetries,
|
|
1141
|
+
total_failovers: totalFailovers,
|
|
1142
|
+
last_error_class: errorClass.class
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
const decision = retryPolicy.decide(
|
|
1146
|
+
{
|
|
1147
|
+
error_class: errorClass,
|
|
1148
|
+
detection_mode: "direct",
|
|
1149
|
+
confidence: "high",
|
|
1150
|
+
raw_signal: { detection_mode: "direct" }
|
|
1151
|
+
},
|
|
1152
|
+
retry
|
|
1153
|
+
);
|
|
1154
|
+
if (decision.action === "exhausted") {
|
|
1155
|
+
log?.info(
|
|
1156
|
+
{ event: "retry_budget_exhausted", target_id: targetId, failover_no: failoverNo },
|
|
1157
|
+
"Retry budget exhausted \u2014 failing over"
|
|
1158
|
+
);
|
|
1159
|
+
attempts.push({
|
|
1160
|
+
target_id: targetId,
|
|
1161
|
+
attempt_no: attemptNo,
|
|
1162
|
+
failover_no: failoverNo,
|
|
1163
|
+
outcome: "failover",
|
|
1164
|
+
error_class: errorClass.class,
|
|
1165
|
+
latency_ms: attemptResult.latency_ms,
|
|
1166
|
+
selection
|
|
1167
|
+
});
|
|
1168
|
+
failedTargets.add(targetId);
|
|
1169
|
+
totalFailovers++;
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
attempts.push({
|
|
1173
|
+
target_id: targetId,
|
|
1174
|
+
attempt_no: attemptNo,
|
|
1175
|
+
failover_no: failoverNo,
|
|
1176
|
+
outcome: "retry",
|
|
1177
|
+
error_class: errorClass.class,
|
|
1178
|
+
latency_ms: attemptResult.latency_ms,
|
|
1179
|
+
selection
|
|
1180
|
+
});
|
|
1181
|
+
totalRetries++;
|
|
1182
|
+
if (decision.action === "retry") {
|
|
1183
|
+
log?.info(
|
|
1184
|
+
{ event: "retry", target_id: targetId, attempt_no: attemptNo, error_class: errorClass.class, delay_ms: decision.delay_ms },
|
|
1185
|
+
"Retrying after delay"
|
|
1186
|
+
);
|
|
1187
|
+
await new Promise(
|
|
1188
|
+
(resolve) => setTimeout(resolve, decision.delay_ms)
|
|
1189
|
+
);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
log?.warn(
|
|
1194
|
+
{ event: "failover_budget_exhausted", total_retries: totalRetries, total_failovers: totalFailovers },
|
|
1195
|
+
"Failover budget exhausted"
|
|
1196
|
+
);
|
|
1197
|
+
return {
|
|
1198
|
+
outcome: "failure",
|
|
1199
|
+
reason: "failover budget exhausted",
|
|
1200
|
+
attempts,
|
|
1201
|
+
total_retries: totalRetries,
|
|
1202
|
+
total_failovers: totalFailovers,
|
|
1203
|
+
last_error_class: lastErrorClass
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
});
|
|
1207
|
+
|
|
1208
|
+
// src/operator/reload.ts
|
|
1209
|
+
var computeConfigDiff = (oldConfig, newConfig) => {
|
|
1210
|
+
const oldTargets = oldConfig.targets ?? [];
|
|
1211
|
+
const newTargets = newConfig.targets ?? [];
|
|
1212
|
+
const oldIds = new Set(oldTargets.map((t) => t.target_id));
|
|
1213
|
+
const newIds = new Set(newTargets.map((t) => t.target_id));
|
|
1214
|
+
const oldMap = new Map(oldTargets.map((t) => [t.target_id, t]));
|
|
1215
|
+
const added = newTargets.filter((t) => !oldIds.has(t.target_id));
|
|
1216
|
+
const removed = oldTargets.filter((t) => !newIds.has(t.target_id)).map((t) => t.target_id);
|
|
1217
|
+
const modified = newTargets.filter((t) => {
|
|
1218
|
+
if (!oldIds.has(t.target_id)) {
|
|
1219
|
+
return false;
|
|
1220
|
+
}
|
|
1221
|
+
const old = oldMap.get(t.target_id);
|
|
1222
|
+
if (!old) {
|
|
1223
|
+
return false;
|
|
1224
|
+
}
|
|
1225
|
+
return hasConfigChanged(old, t);
|
|
1226
|
+
});
|
|
1227
|
+
return { added, removed, modified };
|
|
1228
|
+
};
|
|
1229
|
+
var hasConfigChanged = (oldTarget, newTarget) => {
|
|
1230
|
+
if (oldTarget.profile !== newTarget.profile) return true;
|
|
1231
|
+
if (oldTarget.enabled !== newTarget.enabled) return true;
|
|
1232
|
+
if (oldTarget.operator_priority !== newTarget.operator_priority) return true;
|
|
1233
|
+
if (JSON.stringify(oldTarget.policy_tags) !== JSON.stringify(newTarget.policy_tags)) return true;
|
|
1234
|
+
if (JSON.stringify(oldTarget.capabilities) !== JSON.stringify(newTarget.capabilities)) return true;
|
|
1235
|
+
if (oldTarget.retry_budget !== newTarget.retry_budget) return true;
|
|
1236
|
+
if (oldTarget.failover_budget !== newTarget.failover_budget) return true;
|
|
1237
|
+
if (oldTarget.concurrency_limit !== newTarget.concurrency_limit) return true;
|
|
1238
|
+
return false;
|
|
1239
|
+
};
|
|
1240
|
+
var applyConfigDiff = (registry, diff) => {
|
|
1241
|
+
for (const tc of diff.added) {
|
|
1242
|
+
const entry = {
|
|
1243
|
+
target_id: tc.target_id,
|
|
1244
|
+
provider_id: tc.provider_id,
|
|
1245
|
+
profile: tc.profile,
|
|
1246
|
+
endpoint_id: tc.endpoint_id,
|
|
1247
|
+
capabilities: [...tc.capabilities],
|
|
1248
|
+
enabled: tc.enabled,
|
|
1249
|
+
state: "Active",
|
|
1250
|
+
health_score: INITIAL_HEALTH_SCORE,
|
|
1251
|
+
cooldown_until: null,
|
|
1252
|
+
latency_ema_ms: 0,
|
|
1253
|
+
failure_score: 0,
|
|
1254
|
+
operator_priority: tc.operator_priority,
|
|
1255
|
+
policy_tags: [...tc.policy_tags]
|
|
1256
|
+
};
|
|
1257
|
+
registry.addTarget(entry);
|
|
1258
|
+
}
|
|
1259
|
+
for (const id of diff.removed) {
|
|
1260
|
+
registry.updateState(id, "Disabled");
|
|
1261
|
+
}
|
|
1262
|
+
for (const tc of diff.modified) {
|
|
1263
|
+
const existing = registry.getTarget(tc.target_id);
|
|
1264
|
+
if (existing) {
|
|
1265
|
+
const updated = {
|
|
1266
|
+
...existing,
|
|
1267
|
+
provider_id: tc.provider_id,
|
|
1268
|
+
profile: tc.profile,
|
|
1269
|
+
endpoint_id: tc.endpoint_id,
|
|
1270
|
+
capabilities: [...tc.capabilities],
|
|
1271
|
+
enabled: tc.enabled,
|
|
1272
|
+
operator_priority: tc.operator_priority,
|
|
1273
|
+
policy_tags: [...tc.policy_tags]
|
|
1274
|
+
};
|
|
1275
|
+
registry.removeTarget(tc.target_id);
|
|
1276
|
+
registry.addTarget(updated);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
|
|
1281
|
+
// src/operator/commands.ts
|
|
1282
|
+
var createRequestTraceBuffer = (capacity) => {
|
|
1283
|
+
const entries = /* @__PURE__ */ new Map();
|
|
1284
|
+
const order = [];
|
|
1285
|
+
return {
|
|
1286
|
+
record(entry) {
|
|
1287
|
+
if (entries.has(entry.request_id)) {
|
|
1288
|
+
const idx = order.indexOf(entry.request_id);
|
|
1289
|
+
if (idx !== -1) {
|
|
1290
|
+
order.splice(idx, 1);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
while (order.length >= capacity) {
|
|
1294
|
+
const oldest = order.shift();
|
|
1295
|
+
if (oldest !== void 0) {
|
|
1296
|
+
entries.delete(oldest);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
entries.set(entry.request_id, entry);
|
|
1300
|
+
order.push(entry.request_id);
|
|
1301
|
+
},
|
|
1302
|
+
lookup(requestId) {
|
|
1303
|
+
return entries.get(requestId);
|
|
1304
|
+
},
|
|
1305
|
+
size() {
|
|
1306
|
+
return entries.size;
|
|
1307
|
+
}
|
|
1308
|
+
};
|
|
1309
|
+
};
|
|
1310
|
+
var listTargets = (deps) => {
|
|
1311
|
+
const allTargets = deps.registry.getAllTargets();
|
|
1312
|
+
const targets = allTargets.map((t) => {
|
|
1313
|
+
const cb = deps.circuitBreakers.get(t.target_id);
|
|
1314
|
+
const cbState = cb ? cb.state() : "Active";
|
|
1315
|
+
return {
|
|
1316
|
+
target_id: t.target_id,
|
|
1317
|
+
provider_id: t.provider_id,
|
|
1318
|
+
profile: t.profile,
|
|
1319
|
+
state: t.state,
|
|
1320
|
+
health_score: t.health_score,
|
|
1321
|
+
circuit_breaker_state: cbState,
|
|
1322
|
+
cooldown_until: t.cooldown_until,
|
|
1323
|
+
enabled: t.enabled,
|
|
1324
|
+
latency_ema_ms: t.latency_ema_ms,
|
|
1325
|
+
failure_score: t.failure_score,
|
|
1326
|
+
operator_priority: t.operator_priority
|
|
1327
|
+
};
|
|
1328
|
+
});
|
|
1329
|
+
return { targets, count: targets.length };
|
|
1330
|
+
};
|
|
1331
|
+
var targetAction = (deps, targetId, action, newState) => {
|
|
1332
|
+
const updated = deps.registry.updateState(targetId, newState);
|
|
1333
|
+
if (!updated) {
|
|
1334
|
+
return { success: false, target_id: targetId, action, error: "target not found" };
|
|
1335
|
+
}
|
|
1336
|
+
return { success: true, target_id: targetId, action };
|
|
1337
|
+
};
|
|
1338
|
+
var pauseTarget = (deps, targetId) => targetAction(deps, targetId, "pause", "Disabled");
|
|
1339
|
+
var resumeTarget = (deps, targetId) => targetAction(deps, targetId, "resume", "Active");
|
|
1340
|
+
var drainTarget = (deps, targetId) => targetAction(deps, targetId, "drain", "Draining");
|
|
1341
|
+
var disableTarget = (deps, targetId) => targetAction(deps, targetId, "disable", "Disabled");
|
|
1342
|
+
var inspectRequest = (deps, requestId) => {
|
|
1343
|
+
const trace = deps.traceBuffer.lookup(requestId);
|
|
1344
|
+
if (!trace) {
|
|
1345
|
+
return { found: false, request_id: requestId };
|
|
1346
|
+
}
|
|
1347
|
+
return { found: true, request_id: requestId, trace };
|
|
1348
|
+
};
|
|
1349
|
+
var reloadConfig = (deps, rawConfig) => {
|
|
1350
|
+
try {
|
|
1351
|
+
const newConfig = validateConfig(rawConfig);
|
|
1352
|
+
const currentConfig = deps.configRef.current();
|
|
1353
|
+
const diff = computeConfigDiff(currentConfig, newConfig);
|
|
1354
|
+
applyConfigDiff(deps.registry, diff);
|
|
1355
|
+
deps.configRef.swap(newConfig);
|
|
1356
|
+
return {
|
|
1357
|
+
success: true,
|
|
1358
|
+
added: diff.added.map((t) => t.target_id),
|
|
1359
|
+
removed: [...diff.removed],
|
|
1360
|
+
modified: diff.modified.map((t) => t.target_id)
|
|
1361
|
+
};
|
|
1362
|
+
} catch (err) {
|
|
1363
|
+
if (err instanceof ConfigValidationError) {
|
|
1364
|
+
return {
|
|
1365
|
+
success: false,
|
|
1366
|
+
added: [],
|
|
1367
|
+
removed: [],
|
|
1368
|
+
modified: [],
|
|
1369
|
+
diagnostics: err.diagnostics
|
|
1370
|
+
};
|
|
1371
|
+
}
|
|
1372
|
+
throw err;
|
|
1373
|
+
}
|
|
1374
|
+
};
|
|
1375
|
+
|
|
1376
|
+
// src/operator/plugin-tools.ts
|
|
1377
|
+
import { tool } from "@opencode-ai/plugin/tool";
|
|
1378
|
+
var { schema: z3 } = tool;
|
|
1379
|
+
var createOperatorTools = (deps) => ({
|
|
1380
|
+
listTargets: tool({
|
|
1381
|
+
description: "List all routing targets with health scores, states, and circuit breaker status.",
|
|
1382
|
+
args: {},
|
|
1383
|
+
async execute() {
|
|
1384
|
+
deps.logger.info({ op: "listTargets" }, "operator: listTargets");
|
|
1385
|
+
const result = listTargets(deps);
|
|
1386
|
+
return JSON.stringify(result, null, 2);
|
|
1387
|
+
}
|
|
1388
|
+
}),
|
|
1389
|
+
pauseTarget: tool({
|
|
1390
|
+
description: "Pause a target, preventing new requests from being routed to it.",
|
|
1391
|
+
args: { target_id: z3.string().min(1) },
|
|
1392
|
+
async execute(args) {
|
|
1393
|
+
deps.logger.info(
|
|
1394
|
+
{ op: "pauseTarget", target_id: args.target_id },
|
|
1395
|
+
"operator: pauseTarget"
|
|
1396
|
+
);
|
|
1397
|
+
const result = pauseTarget(deps, args.target_id);
|
|
1398
|
+
return JSON.stringify(result, null, 2);
|
|
1399
|
+
}
|
|
1400
|
+
}),
|
|
1401
|
+
resumeTarget: tool({
|
|
1402
|
+
description: "Resume a previously paused or disabled target, allowing new requests.",
|
|
1403
|
+
args: { target_id: z3.string().min(1) },
|
|
1404
|
+
async execute(args) {
|
|
1405
|
+
deps.logger.info(
|
|
1406
|
+
{ op: "resumeTarget", target_id: args.target_id },
|
|
1407
|
+
"operator: resumeTarget"
|
|
1408
|
+
);
|
|
1409
|
+
const result = resumeTarget(deps, args.target_id);
|
|
1410
|
+
return JSON.stringify(result, null, 2);
|
|
1411
|
+
}
|
|
1412
|
+
}),
|
|
1413
|
+
drainTarget: tool({
|
|
1414
|
+
description: "Drain a target, allowing in-flight requests to complete but preventing new ones.",
|
|
1415
|
+
args: { target_id: z3.string().min(1) },
|
|
1416
|
+
async execute(args) {
|
|
1417
|
+
deps.logger.info(
|
|
1418
|
+
{ op: "drainTarget", target_id: args.target_id },
|
|
1419
|
+
"operator: drainTarget"
|
|
1420
|
+
);
|
|
1421
|
+
const result = drainTarget(deps, args.target_id);
|
|
1422
|
+
return JSON.stringify(result, null, 2);
|
|
1423
|
+
}
|
|
1424
|
+
}),
|
|
1425
|
+
disableTarget: tool({
|
|
1426
|
+
description: "Disable a target entirely, removing it from routing.",
|
|
1427
|
+
args: { target_id: z3.string().min(1) },
|
|
1428
|
+
async execute(args) {
|
|
1429
|
+
deps.logger.info(
|
|
1430
|
+
{ op: "disableTarget", target_id: args.target_id },
|
|
1431
|
+
"operator: disableTarget"
|
|
1432
|
+
);
|
|
1433
|
+
const result = disableTarget(deps, args.target_id);
|
|
1434
|
+
return JSON.stringify(result, null, 2);
|
|
1435
|
+
}
|
|
1436
|
+
}),
|
|
1437
|
+
inspectRequest: tool({
|
|
1438
|
+
description: "Inspect a request trace by ID, showing attempts, segments, and outcome.",
|
|
1439
|
+
args: { request_id: z3.string().min(1) },
|
|
1440
|
+
async execute(args) {
|
|
1441
|
+
deps.logger.info(
|
|
1442
|
+
{ op: "inspectRequest", request_id: args.request_id },
|
|
1443
|
+
"operator: inspectRequest"
|
|
1444
|
+
);
|
|
1445
|
+
const result = inspectRequest(deps, args.request_id);
|
|
1446
|
+
return JSON.stringify(result, null, 2);
|
|
1447
|
+
}
|
|
1448
|
+
}),
|
|
1449
|
+
reloadConfig: tool({
|
|
1450
|
+
description: "Reload routing configuration with diff-apply. Validates new config before applying.",
|
|
1451
|
+
args: { config: z3.record(z3.string(), z3.unknown()) },
|
|
1452
|
+
async execute(args) {
|
|
1453
|
+
deps.logger.info({ op: "reloadConfig" }, "operator: reloadConfig");
|
|
1454
|
+
const result = reloadConfig(deps, args.config);
|
|
1455
|
+
return JSON.stringify(result, null, 2);
|
|
1456
|
+
}
|
|
1457
|
+
})
|
|
1458
|
+
});
|
|
1459
|
+
|
|
1460
|
+
// src/profiles/store.ts
|
|
1461
|
+
import { readFile, writeFile, rename, mkdir } from "fs/promises";
|
|
1462
|
+
import { homedir } from "os";
|
|
1463
|
+
import { join, dirname } from "path";
|
|
1464
|
+
var PROFILES_DIR = join(homedir(), ".local", "share", "o-switcher");
|
|
1465
|
+
var PROFILES_PATH = join(PROFILES_DIR, "profiles.json");
|
|
1466
|
+
var loadProfiles = async (path = PROFILES_PATH) => {
|
|
1467
|
+
try {
|
|
1468
|
+
const content = await readFile(path, "utf-8");
|
|
1469
|
+
return JSON.parse(content);
|
|
1470
|
+
} catch (err) {
|
|
1471
|
+
const code = err.code;
|
|
1472
|
+
if (code === "ENOENT") {
|
|
1473
|
+
return {};
|
|
1474
|
+
}
|
|
1475
|
+
throw err;
|
|
1476
|
+
}
|
|
1477
|
+
};
|
|
1478
|
+
var saveProfiles = async (store, path = PROFILES_PATH, logger) => {
|
|
1479
|
+
const dir = dirname(path);
|
|
1480
|
+
await mkdir(dir, { recursive: true });
|
|
1481
|
+
const tmpPath = `${path}.tmp`;
|
|
1482
|
+
await writeFile(tmpPath, JSON.stringify(store, null, 2), "utf-8");
|
|
1483
|
+
await rename(tmpPath, path);
|
|
1484
|
+
logger?.info({ path }, "Profiles saved to disk");
|
|
1485
|
+
};
|
|
1486
|
+
var credentialsMatch = (a, b) => {
|
|
1487
|
+
if (a.type !== b.type) return false;
|
|
1488
|
+
if (a.type === "api-key" && b.type === "api-key") {
|
|
1489
|
+
return a.key === b.key;
|
|
1490
|
+
}
|
|
1491
|
+
if (a.type === "oauth" && b.type === "oauth") {
|
|
1492
|
+
return a.refresh === b.refresh && a.access === b.access && a.expires === b.expires && a.accountId === b.accountId;
|
|
1493
|
+
}
|
|
1494
|
+
return false;
|
|
1495
|
+
};
|
|
1496
|
+
var addProfile = (store, provider, credentials) => {
|
|
1497
|
+
const isDuplicate = Object.values(store).some(
|
|
1498
|
+
(entry2) => entry2.provider === provider && credentialsMatch(entry2.credentials, credentials)
|
|
1499
|
+
);
|
|
1500
|
+
if (isDuplicate) {
|
|
1501
|
+
return store;
|
|
1502
|
+
}
|
|
1503
|
+
const id = nextProfileId(store, provider);
|
|
1504
|
+
const entry = {
|
|
1505
|
+
id,
|
|
1506
|
+
provider,
|
|
1507
|
+
type: credentials.type,
|
|
1508
|
+
credentials,
|
|
1509
|
+
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1510
|
+
};
|
|
1511
|
+
return { ...store, [id]: entry };
|
|
1512
|
+
};
|
|
1513
|
+
var removeProfile = (store, id) => {
|
|
1514
|
+
if (store[id] === void 0) {
|
|
1515
|
+
return { store, removed: false };
|
|
1516
|
+
}
|
|
1517
|
+
const { [id]: _removed, ...rest } = store;
|
|
1518
|
+
return { store: rest, removed: true };
|
|
1519
|
+
};
|
|
1520
|
+
var listProfiles = (store) => {
|
|
1521
|
+
return Object.values(store).sort(
|
|
1522
|
+
(a, b) => new Date(a.created).getTime() - new Date(b.created).getTime()
|
|
1523
|
+
);
|
|
1524
|
+
};
|
|
1525
|
+
var nextProfileId = (store, provider) => {
|
|
1526
|
+
const prefix = `${provider}-`;
|
|
1527
|
+
const maxN = Object.keys(store).filter((key) => key.startsWith(prefix)).map((key) => Number(key.slice(prefix.length))).filter((n) => !Number.isNaN(n)).reduce((max, n) => Math.max(max, n), 0);
|
|
1528
|
+
return `${provider}-${maxN + 1}`;
|
|
1529
|
+
};
|
|
1530
|
+
|
|
1531
|
+
// src/profiles/watcher.ts
|
|
1532
|
+
import { readFile as readFile2, watch } from "fs/promises";
|
|
1533
|
+
import { homedir as homedir2 } from "os";
|
|
1534
|
+
import { join as join2, dirname as dirname2 } from "path";
|
|
1535
|
+
var AUTH_JSON_PATH = join2(
|
|
1536
|
+
homedir2(),
|
|
1537
|
+
".local",
|
|
1538
|
+
"share",
|
|
1539
|
+
"opencode",
|
|
1540
|
+
"auth.json"
|
|
1541
|
+
);
|
|
1542
|
+
var DEBOUNCE_MS = 100;
|
|
1543
|
+
var readAuthJson = async (path) => {
|
|
1544
|
+
try {
|
|
1545
|
+
const content = await readFile2(path, "utf-8");
|
|
1546
|
+
return JSON.parse(content);
|
|
1547
|
+
} catch {
|
|
1548
|
+
return {};
|
|
1549
|
+
}
|
|
1550
|
+
};
|
|
1551
|
+
var toCredential = (entry) => {
|
|
1552
|
+
if (entry.type === "oauth" || entry.refresh !== void 0 && entry.access !== void 0) {
|
|
1553
|
+
return {
|
|
1554
|
+
type: "oauth",
|
|
1555
|
+
refresh: String(entry.refresh ?? ""),
|
|
1556
|
+
access: String(entry.access ?? ""),
|
|
1557
|
+
expires: Number(entry.expires ?? 0),
|
|
1558
|
+
accountId: entry.accountId !== void 0 ? String(entry.accountId) : void 0
|
|
1559
|
+
};
|
|
1560
|
+
}
|
|
1561
|
+
return {
|
|
1562
|
+
type: "api-key",
|
|
1563
|
+
key: String(entry.key ?? "")
|
|
1564
|
+
};
|
|
1565
|
+
};
|
|
1566
|
+
var entriesEqual = (a, b) => {
|
|
1567
|
+
if (a === void 0 && b === void 0) return true;
|
|
1568
|
+
if (a === void 0 || b === void 0) return false;
|
|
1569
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
1570
|
+
};
|
|
1571
|
+
var createAuthWatcher = (options) => {
|
|
1572
|
+
const authPath = options?.authJsonPath ?? AUTH_JSON_PATH;
|
|
1573
|
+
const profPath = options?.profilesPath ?? PROFILES_PATH;
|
|
1574
|
+
const log = options?.logger?.child({ component: "profile-watcher" });
|
|
1575
|
+
let lastKnownAuth = {};
|
|
1576
|
+
let abortController = null;
|
|
1577
|
+
let debounceTimer = null;
|
|
1578
|
+
let watchPromise = null;
|
|
1579
|
+
const processChange = async () => {
|
|
1580
|
+
const newAuth = await readAuthJson(authPath);
|
|
1581
|
+
let store = await loadProfiles(profPath);
|
|
1582
|
+
let changed = false;
|
|
1583
|
+
for (const [provider, entry] of Object.entries(newAuth)) {
|
|
1584
|
+
if (!entry) continue;
|
|
1585
|
+
const previousEntry = lastKnownAuth[provider];
|
|
1586
|
+
if (previousEntry !== void 0 && !entriesEqual(previousEntry, entry)) {
|
|
1587
|
+
log?.info({ provider, action: "credential_changed" }, "Credential change detected \u2014 saving previous credential");
|
|
1588
|
+
const prevCredential = toCredential(previousEntry);
|
|
1589
|
+
const newStore = addProfile(store, provider, prevCredential);
|
|
1590
|
+
if (newStore !== store) {
|
|
1591
|
+
store = newStore;
|
|
1592
|
+
changed = true;
|
|
1593
|
+
}
|
|
1594
|
+
} else if (previousEntry === void 0) {
|
|
1595
|
+
log?.info({ provider, action: "new_provider" }, "New provider detected");
|
|
1596
|
+
const credential = toCredential(entry);
|
|
1597
|
+
const newStore = addProfile(store, provider, credential);
|
|
1598
|
+
if (newStore !== store) {
|
|
1599
|
+
store = newStore;
|
|
1600
|
+
changed = true;
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
if (changed) {
|
|
1605
|
+
await saveProfiles(store, profPath);
|
|
1606
|
+
log?.info({ profiles_saved: true }, "Profiles saved to disk");
|
|
1607
|
+
}
|
|
1608
|
+
lastKnownAuth = newAuth;
|
|
1609
|
+
};
|
|
1610
|
+
const onFileChange = () => {
|
|
1611
|
+
if (debounceTimer !== null) {
|
|
1612
|
+
clearTimeout(debounceTimer);
|
|
1613
|
+
}
|
|
1614
|
+
debounceTimer = setTimeout(() => {
|
|
1615
|
+
debounceTimer = null;
|
|
1616
|
+
processChange().catch((err) => {
|
|
1617
|
+
log?.warn({ err }, "Error processing auth change");
|
|
1618
|
+
});
|
|
1619
|
+
}, DEBOUNCE_MS);
|
|
1620
|
+
};
|
|
1621
|
+
const start = async () => {
|
|
1622
|
+
const currentAuth = await readAuthJson(authPath);
|
|
1623
|
+
const currentStore = await loadProfiles(profPath);
|
|
1624
|
+
log?.info({ providers: Object.keys(currentAuth) }, "Auth watcher started");
|
|
1625
|
+
if (Object.keys(currentStore).length === 0 && Object.keys(currentAuth).length > 0) {
|
|
1626
|
+
let store = {};
|
|
1627
|
+
for (const [provider, entry] of Object.entries(currentAuth)) {
|
|
1628
|
+
if (!entry) continue;
|
|
1629
|
+
const credential = toCredential(entry);
|
|
1630
|
+
store = addProfile(store, provider, credential);
|
|
1631
|
+
}
|
|
1632
|
+
await saveProfiles(store, profPath);
|
|
1633
|
+
log?.info({ profiles_initialized: Object.keys(store).length }, "Initialized profiles from auth.json");
|
|
1634
|
+
}
|
|
1635
|
+
lastKnownAuth = currentAuth;
|
|
1636
|
+
abortController = new AbortController();
|
|
1637
|
+
const parentDir = dirname2(authPath);
|
|
1638
|
+
watchPromise = (async () => {
|
|
1639
|
+
try {
|
|
1640
|
+
const watcher = watch(parentDir, {
|
|
1641
|
+
signal: abortController.signal
|
|
1642
|
+
});
|
|
1643
|
+
for await (const event of watcher) {
|
|
1644
|
+
if (event.filename === "auth.json" || event.filename === null) {
|
|
1645
|
+
onFileChange();
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
} catch (err) {
|
|
1649
|
+
const name = err.name;
|
|
1650
|
+
if (name !== "AbortError") {
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
})();
|
|
1654
|
+
};
|
|
1655
|
+
const stop = () => {
|
|
1656
|
+
if (debounceTimer !== null) {
|
|
1657
|
+
clearTimeout(debounceTimer);
|
|
1658
|
+
debounceTimer = null;
|
|
1659
|
+
}
|
|
1660
|
+
if (abortController !== null) {
|
|
1661
|
+
abortController.abort();
|
|
1662
|
+
abortController = null;
|
|
1663
|
+
}
|
|
1664
|
+
watchPromise = null;
|
|
1665
|
+
log?.info("Auth watcher stopped");
|
|
1666
|
+
};
|
|
1667
|
+
return { start, stop };
|
|
1668
|
+
};
|
|
1669
|
+
|
|
1670
|
+
// src/profiles/tools.ts
|
|
1671
|
+
import { tool as tool2 } from "@opencode-ai/plugin/tool";
|
|
1672
|
+
var { schema: z4 } = tool2;
|
|
1673
|
+
var createProfileTools = (options) => ({
|
|
1674
|
+
profilesList: tool2({
|
|
1675
|
+
description: "List all saved auth profiles with provider, type, and creation date.",
|
|
1676
|
+
args: {},
|
|
1677
|
+
async execute() {
|
|
1678
|
+
const store = await loadProfiles(options?.profilesPath);
|
|
1679
|
+
const profiles = listProfiles(store);
|
|
1680
|
+
const result = profiles.map((entry) => ({
|
|
1681
|
+
id: entry.id,
|
|
1682
|
+
provider: entry.provider,
|
|
1683
|
+
type: entry.type,
|
|
1684
|
+
created: entry.created
|
|
1685
|
+
}));
|
|
1686
|
+
return JSON.stringify(result, null, 2);
|
|
1687
|
+
}
|
|
1688
|
+
}),
|
|
1689
|
+
profilesRemove: tool2({
|
|
1690
|
+
description: "Remove a saved auth profile by ID.",
|
|
1691
|
+
args: { id: z4.string().min(1) },
|
|
1692
|
+
async execute(args) {
|
|
1693
|
+
const store = await loadProfiles(options?.profilesPath);
|
|
1694
|
+
const { store: newStore, removed } = removeProfile(store, args.id);
|
|
1695
|
+
if (removed) {
|
|
1696
|
+
await saveProfiles(newStore, options?.profilesPath);
|
|
1697
|
+
return JSON.stringify({ success: true, id: args.id });
|
|
1698
|
+
}
|
|
1699
|
+
return JSON.stringify({
|
|
1700
|
+
success: false,
|
|
1701
|
+
id: args.id,
|
|
1702
|
+
error: "Profile not found"
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
})
|
|
1706
|
+
});
|
|
1707
|
+
|
|
1708
|
+
export {
|
|
1709
|
+
DEFAULT_RETRY_BUDGET,
|
|
1710
|
+
DEFAULT_FAILOVER_BUDGET,
|
|
1711
|
+
DEFAULT_BACKOFF_BASE_MS,
|
|
1712
|
+
DEFAULT_BACKOFF_MULTIPLIER,
|
|
1713
|
+
DEFAULT_BACKOFF_MAX_MS,
|
|
1714
|
+
DEFAULT_BACKOFF_JITTER,
|
|
1715
|
+
DEFAULT_RETRY,
|
|
1716
|
+
DEFAULT_TIMEOUT_MS,
|
|
1717
|
+
BackoffConfigSchema,
|
|
1718
|
+
TargetConfigSchema,
|
|
1719
|
+
SwitcherConfigSchema,
|
|
1720
|
+
ConfigValidationError,
|
|
1721
|
+
validateConfig,
|
|
1722
|
+
discoverTargets,
|
|
1723
|
+
discoverTargetsFromProfiles,
|
|
1724
|
+
INITIAL_HEALTH_SCORE,
|
|
1725
|
+
DEFAULT_ALPHA,
|
|
1726
|
+
updateHealthScore,
|
|
1727
|
+
updateLatencyEma,
|
|
1728
|
+
TargetRegistry,
|
|
1729
|
+
createRegistry,
|
|
1730
|
+
TARGET_STATES,
|
|
1731
|
+
REDACT_PATHS,
|
|
1732
|
+
createAuditLogger,
|
|
1733
|
+
createRequestLogger,
|
|
1734
|
+
generateCorrelationId,
|
|
1735
|
+
ErrorClassSchema,
|
|
1736
|
+
isRetryable,
|
|
1737
|
+
getTargetStateTransition,
|
|
1738
|
+
DEFAULT_BACKOFF_PARAMS,
|
|
1739
|
+
computeBackoffMs,
|
|
1740
|
+
createRetryPolicy,
|
|
1741
|
+
EXCLUSION_REASONS,
|
|
1742
|
+
ADMISSION_RESULTS,
|
|
1743
|
+
createRoutingEventBus,
|
|
1744
|
+
DualBreaker,
|
|
1745
|
+
createCircuitBreaker,
|
|
1746
|
+
createConcurrencyTracker,
|
|
1747
|
+
normalizeLatency,
|
|
1748
|
+
computeScore,
|
|
1749
|
+
getExclusionReason,
|
|
1750
|
+
selectTarget,
|
|
1751
|
+
computeCooldownMs,
|
|
1752
|
+
createCooldownManager,
|
|
1753
|
+
checkHardRejects,
|
|
1754
|
+
createAdmissionController,
|
|
1755
|
+
createLogSubscriber,
|
|
1756
|
+
createFailoverOrchestrator,
|
|
1757
|
+
computeConfigDiff,
|
|
1758
|
+
applyConfigDiff,
|
|
1759
|
+
createRequestTraceBuffer,
|
|
1760
|
+
listTargets,
|
|
1761
|
+
pauseTarget,
|
|
1762
|
+
resumeTarget,
|
|
1763
|
+
drainTarget,
|
|
1764
|
+
disableTarget,
|
|
1765
|
+
inspectRequest,
|
|
1766
|
+
reloadConfig,
|
|
1767
|
+
createOperatorTools,
|
|
1768
|
+
loadProfiles,
|
|
1769
|
+
saveProfiles,
|
|
1770
|
+
addProfile,
|
|
1771
|
+
removeProfile,
|
|
1772
|
+
listProfiles,
|
|
1773
|
+
nextProfileId,
|
|
1774
|
+
createAuthWatcher,
|
|
1775
|
+
createProfileTools
|
|
1776
|
+
};
|
|
1777
|
+
//# sourceMappingURL=chunk-BTDKGS7P.js.map
|