@genee/omp-opsx-addon 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1246 @@
1
+ /**
2
+ * Adaptive per-provider concurrency: signal collection + control-loop
3
+ * orchestration.
4
+ *
5
+ * The pure AIMD state machine and the host-signal parsers live in
6
+ * `concurrency-tuner.ts`; the config/lease write path lives in
7
+ * `concurrency-writer.ts`. This module only wires the two together:
8
+ *
9
+ * host render tick (the existing second-level tick; absolute-time gated by
10
+ * `concurrency_eval_interval_ms` — this module MUST NOT create a timer)
11
+ * ├─ ① explicit rate-limit / timeout events, delivered in-process
12
+ * │ (`noteExplicitFailure`, reusing `error-scan`'s `hasRateLimitError`)
13
+ * │ and aggregated across processes through the injected Redis port
14
+ * ├─ ② queue waits from the incremental host-log reader (every pid under
15
+ * │ the same config root; guardrail input only, never a bad event)
16
+ * └─ ③④ generation-duration / throughput from `agent.db:model_perf`
17
+ *
18
+ * Every IO source is an injected port (fs / model_perf / redis / settings /
19
+ * lease), so the module runs with zero real IO in tests; the production
20
+ * implementations live here and are wired by the assembly layer.
21
+ */
22
+
23
+ import { Database } from 'bun:sqlite';
24
+ import { createHash } from 'node:crypto';
25
+ import { closeSync, openSync, readSync, readdirSync, statSync } from 'node:fs';
26
+ import { homedir } from 'node:os';
27
+ import { dirname, join, resolve as resolvePath } from 'node:path';
28
+ import {
29
+ BLOCKED_MESSAGE,
30
+ WAIT_COMPLETED_MESSAGE,
31
+ TUNER_DEFAULTS,
32
+ clampLimit,
33
+ createProviderState,
34
+ diffModelPerf,
35
+ evaluateWindow,
36
+ formatConcurrencySummary,
37
+ parseInFlightLogLines,
38
+ queueWaitPercentiles,
39
+ updatePerfBaseline,
40
+ type BadEventKind,
41
+ type ModelPerfRow,
42
+ type PerfBaseline,
43
+ type ProviderConcurrencyState,
44
+ type ProviderPerfWindow,
45
+ type QueueWaitSample,
46
+ type TunerOptions,
47
+ type WindowSignals,
48
+ } from './concurrency-tuner.js';
49
+ import {
50
+ applyLimit,
51
+ buildMergedMap,
52
+ createLease,
53
+ readManagedLimits,
54
+ resolveTakeoverStart,
55
+ shouldWriteLimit,
56
+ writeManagedLimit,
57
+ type LeaseHandle,
58
+ type LeaseMode,
59
+ type SettingsPort,
60
+ type WriteManagedLimitResult,
61
+ } from './concurrency-writer.js';
62
+ import {
63
+ STATE_FILE_NAME,
64
+ applyDecayStep,
65
+ loadConcurrencyState,
66
+ restoreLimit,
67
+ saveConcurrencyState,
68
+ stopFailSafe,
69
+ toStateFile,
70
+ type ConcurrencyStateFile,
71
+ type PersistedProviderState,
72
+ } from './concurrency-state.js';
73
+ import { hasRateLimitError } from './error-scan.js';
74
+ import { canonicalizeProvider } from './usage-resolver.js';
75
+ import type { RedisReply } from './usage-redis-client.js';
76
+
77
+ /** Warn sink; the assembly layer passes the plugin logger. */
78
+ export type ConcurrencyWarn = (message: string) => void;
79
+
80
+ /** Host log file name pattern (`omp.<date>.<pid>.log`). */
81
+ export const LOG_FILE_PATTERN = /^omp\..*\.log$/;
82
+ /** Newest log files read per tick (all pids under the same config root). */
83
+ export const DEFAULT_MAX_LOG_FILES = 4;
84
+ /** Tail probe size for the runtime debug-signal self-check. */
85
+ export const DEFAULT_LOG_PROBE_BYTES = 64 * 1024;
86
+ /** Hard cap on an unterminated log line before its bytes are dropped. */
87
+ export const MAX_LOG_LINE_BYTES = 1 << 20;
88
+ /** Recent log lines retained so a `blocked`/`wait completed` pair can span ticks. */
89
+ export const MAX_BUFFERED_LOG_LINES = 5_000;
90
+ /** File-lease name under the config root (Redis lease preferred). */
91
+ export const LEASE_LOCK_FILENAME = 'opsx-concurrency-tuner.lock';
92
+ /** `concurrency_write_min_interval_ms` default — minimum spacing between writes. */
93
+ export const DEFAULT_WRITE_MIN_INTERVAL_MS = 20_000;
94
+ /** Redis counter key namespace for cross-process ① aggregation. */
95
+ export const RATE_LIMIT_REDIS_PREFIX = 'opsx:concurrency:rl';
96
+ /** Aggregation counter TTL, in evaluation intervals. */
97
+ export const AGGREGATION_TTL_INTERVALS = 4;
98
+
99
+ // ── Injected ports ──────────────────────────────────────────────────────────
100
+
101
+ /** Filesystem seam for the log reader (synchronous; the tick is sync-friendly). */
102
+ export interface ConcurrencyFsPort {
103
+ /** Directory entry names; throws when the directory is unreadable. */
104
+ readdir(dir: string): string[];
105
+ /** Size/mtime (+ inode when the platform exposes one); throws when unreadable. */
106
+ stat(path: string): { size: number; mtimeMs: number; ino?: number };
107
+ /** UTF-8 text for byte range `[start, end)`; throws when unreadable. */
108
+ readRange(path: string, start: number, end: number): string;
109
+ }
110
+
111
+ /** Read-only `model_perf` source (cumulative counters). */
112
+ export interface ModelPerfPort {
113
+ /** Cumulative rows; throws when the DB is missing/locked/without the table. */
114
+ readModelPerf(): ModelPerfRow[];
115
+ }
116
+
117
+ /** Minimal Redis seam: the batched-command surface of `RedisClient.pipeline()`. */
118
+ export interface ConcurrencyRedisPort {
119
+ pipeline(commands: string[][]): Promise<RedisReply[]>;
120
+ }
121
+
122
+ /** Settings hot-apply seam (`pi.pi.settings`); `override` is whole-key. */
123
+ export type { SettingsPort };
124
+
125
+ /** Lease surface the loop drives, as produced by the writer's `createLease`. */
126
+ export type ConcurrencyLease = LeaseHandle;
127
+
128
+ export interface ConcurrencyLeaseArgs {
129
+ redis: ConcurrencyRedisPort | null;
130
+ lockPath: string;
131
+ owner: string;
132
+ ttlMs: number;
133
+ now: () => number;
134
+ }
135
+
136
+ export type ConcurrencyLeaseFactory = (args: ConcurrencyLeaseArgs) => Promise<ConcurrencyLease>;
137
+
138
+ const defaultLeaseFactory: ConcurrencyLeaseFactory = (args) =>
139
+ createLease({
140
+ redis: args.redis,
141
+ lockPath: args.lockPath,
142
+ owner: args.owner,
143
+ ttlMs: args.ttlMs,
144
+ now: args.now,
145
+ });
146
+
147
+ /** Production fs port backed by `node:fs`. */
148
+ export function createNodeFsPort(): ConcurrencyFsPort {
149
+ return {
150
+ readdir: (dir) => readdirSync(dir),
151
+ stat: (path) => {
152
+ const st = statSync(path);
153
+ return {
154
+ size: st.size,
155
+ mtimeMs: st.mtimeMs,
156
+ ino: typeof st.ino === 'number' && st.ino > 0 ? st.ino : undefined,
157
+ };
158
+ },
159
+ readRange: (path, start, end) => {
160
+ const length = Math.max(0, end - start);
161
+ if (length === 0) return '';
162
+ const fd = openSync(path, 'r');
163
+ try {
164
+ const buf = Buffer.allocUnsafe(length);
165
+ const read = readSync(fd, buf, 0, length, start);
166
+ return buf.subarray(0, read).toString('utf8');
167
+ } finally {
168
+ closeSync(fd);
169
+ }
170
+ },
171
+ };
172
+ }
173
+
174
+ /** Read-only `bun:sqlite` port over `<agentDir>/agent.db`. */
175
+ export function createSqliteModelPerfPort(dbPath: string): ModelPerfPort {
176
+ return {
177
+ readModelPerf(): ModelPerfRow[] {
178
+ const db = new Database(dbPath, { readonly: true });
179
+ try {
180
+ const rows = db
181
+ .query('SELECT model_key, samples, output_tokens, gen_ms FROM model_perf')
182
+ .all() as Array<Record<string, unknown>>;
183
+ const count = (value: unknown): number => {
184
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
185
+ if (typeof value === 'bigint') return Number(value);
186
+ return 0;
187
+ };
188
+ const out: ModelPerfRow[] = [];
189
+ for (const row of rows) {
190
+ const modelKey = row.model_key;
191
+ if (typeof modelKey !== 'string' || modelKey === '') continue;
192
+ out.push({
193
+ modelKey,
194
+ samples: count(row.samples),
195
+ outputTokens: count(row.output_tokens),
196
+ genMs: count(row.gen_ms),
197
+ });
198
+ }
199
+ return out;
200
+ } finally {
201
+ db.close();
202
+ }
203
+ },
204
+ };
205
+ }
206
+
207
+ // ── Path derivation (tasks 3.2 / 3.3) ───────────────────────────────────────
208
+
209
+ /** `settings.getAgentDir()` when present, else `<homedir>/.omp/agent`. */
210
+ export function deriveAgentDir(agentDir: string | null | undefined): string {
211
+ return agentDir && agentDir.length > 0 ? agentDir : join(homedir(), '.omp', 'agent');
212
+ }
213
+
214
+ /** Host log directory for an agent dir (`<agentDir>/../logs`). */
215
+ export function deriveLogDir(agentDir: string | null | undefined): string {
216
+ return join(deriveAgentDir(agentDir), '..', 'logs');
217
+ }
218
+
219
+ /** `model_perf` DB path for an agent dir. */
220
+ export function deriveDbPath(agentDir: string | null | undefined): string {
221
+ return join(deriveAgentDir(agentDir), 'agent.db');
222
+ }
223
+
224
+ /** Stable per-config-root hash used in Redis keys. */
225
+ export function configRootHash(configPath: string): string {
226
+ return createHash('sha1').update(resolvePath(dirname(configPath))).digest('hex').slice(0, 12);
227
+ }
228
+
229
+ // ── Incremental host-log reader (signal ②) ──────────────────────────────────
230
+
231
+ export interface IncrementalLogSourceOptions {
232
+ maxFiles?: number;
233
+ warn?: ConcurrencyWarn;
234
+ }
235
+
236
+ interface TrackedLogFile {
237
+ /** Byte offset of the next unread byte. */
238
+ offset: number;
239
+ ino: number | null;
240
+ }
241
+
242
+ /**
243
+ * Byte-offset tailer over the host's `omp.*.log` files.
244
+ *
245
+ * The reader is shared by every pid under the same config root, so a wait
246
+ * blocked in one process is visible here. A file seen for the first time is
247
+ * tailed from its current end (joining mid-stream must not replay stale
248
+ * `blocked` lines and pair them with fresh completions); a rotated (inode
249
+ * change) or truncated (size < offset) file is re-read from byte 0. Failure to
250
+ * list/stat/read is warned once and skipped — collection never throws.
251
+ */
252
+ export class IncrementalLogSource {
253
+ private readonly tracked = new Map<string, TrackedLogFile>();
254
+ private readonly warned = new Set<string>();
255
+ private readonly maxFiles: number;
256
+ private readonly warn: ConcurrencyWarn;
257
+
258
+ constructor(
259
+ private readonly fs: ConcurrencyFsPort,
260
+ private readonly logDir: string,
261
+ options: IncrementalLogSourceOptions = {},
262
+ ) {
263
+ this.maxFiles = Math.max(1, options.maxFiles ?? DEFAULT_MAX_LOG_FILES);
264
+ this.warn = options.warn ?? (() => {});
265
+ }
266
+
267
+ /** New complete lines from every selected log file since the previous call. */
268
+ readNewLines(): string[] {
269
+ const lines: string[] = [];
270
+ for (const file of this.listLogFiles()) {
271
+ for (const line of this.readFile(file)) lines.push(line);
272
+ }
273
+ return lines;
274
+ }
275
+
276
+ /**
277
+ * Runtime self-check (tasks 3.2): does the newest log currently carry a
278
+ * `level=debug` in-flight `blocked` / `wait completed` line? Signal ② is
279
+ * empty when the host raises the log level or turns debug off, so the
280
+ * assembly runs this once and surfaces the outcome in the summary.
281
+ */
282
+ probeDebugInFlightLines(probeBytes = DEFAULT_LOG_PROBE_BYTES): boolean {
283
+ const files = this.listLogFiles();
284
+ if (files.length === 0) return false;
285
+ const newest = files.reduce((a, b) => (b.mtimeMs > a.mtimeMs ? b : a));
286
+ const start = Math.max(0, newest.size - Math.max(1, probeBytes));
287
+ let text: string;
288
+ try {
289
+ text = this.fs.readRange(newest.path, start, newest.size);
290
+ } catch {
291
+ this.warnOnce(`probe:${newest.path}`, `[omp-opsx-addon] concurrency: log self-check cannot read ${newest.path}`);
292
+ return false;
293
+ }
294
+ for (const raw of text.split('\n')) {
295
+ const line = raw.trim();
296
+ if (line === '') continue;
297
+ let rec: Record<string, unknown>;
298
+ try {
299
+ const parsed: unknown = JSON.parse(line);
300
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) continue;
301
+ rec = parsed as Record<string, unknown>;
302
+ } catch {
303
+ continue;
304
+ }
305
+ if (rec.level !== 'debug') continue;
306
+ if (rec.message === BLOCKED_MESSAGE || rec.message === WAIT_COMPLETED_MESSAGE) return true;
307
+ }
308
+ return false;
309
+ }
310
+
311
+ private listLogFiles(): Array<{ path: string; size: number; mtimeMs: number; ino?: number }> {
312
+ let names: string[];
313
+ try {
314
+ names = this.fs.readdir(this.logDir);
315
+ } catch (err) {
316
+ this.warnOnce('readdir', `[omp-opsx-addon] concurrency: log dir unreadable (${this.logDir}): ${describe(err)}`);
317
+ return [];
318
+ }
319
+ const files: Array<{ path: string; size: number; mtimeMs: number; ino?: number }> = [];
320
+ for (const name of names) {
321
+ if (!LOG_FILE_PATTERN.test(name)) continue;
322
+ const path = join(this.logDir, name);
323
+ try {
324
+ const st = this.fs.stat(path);
325
+ files.push({ path, size: st.size, mtimeMs: st.mtimeMs, ino: st.ino });
326
+ } catch (err) {
327
+ this.warnOnce(`stat:${path}`, `[omp-opsx-addon] concurrency: log file unreadable (${path}): ${describe(err)}`);
328
+ }
329
+ }
330
+ const present = new Set(files.map((f) => f.path));
331
+ for (const path of [...this.tracked.keys()]) {
332
+ if (!present.has(path)) this.tracked.delete(path);
333
+ }
334
+ files.sort((a, b) => b.mtimeMs - a.mtimeMs);
335
+ return files.slice(0, this.maxFiles);
336
+ }
337
+
338
+ private readFile(file: { path: string; size: number; mtimeMs: number; ino?: number }): string[] {
339
+ const ino = typeof file.ino === 'number' ? file.ino : null;
340
+ const tracked = this.tracked.get(file.path);
341
+ if (!tracked) {
342
+ this.tracked.set(file.path, { offset: file.size, ino });
343
+ return [];
344
+ }
345
+ const rotated = ino !== null && tracked.ino !== null && ino !== tracked.ino;
346
+ if (rotated || file.size < tracked.offset) {
347
+ tracked.offset = 0;
348
+ tracked.ino = ino;
349
+ } else if (ino !== null) {
350
+ tracked.ino = ino;
351
+ }
352
+ if (file.size <= tracked.offset) return [];
353
+
354
+ let text: string;
355
+ try {
356
+ text = this.fs.readRange(file.path, tracked.offset, file.size);
357
+ } catch (err) {
358
+ this.warnOnce(`read:${file.path}`, `[omp-opsx-addon] concurrency: log file read failed (${file.path}): ${describe(err)}`);
359
+ return [];
360
+ }
361
+ const lastNewline = text.lastIndexOf('\n');
362
+ if (lastNewline < 0) {
363
+ if (file.size - tracked.offset > MAX_LOG_LINE_BYTES) {
364
+ this.warnOnce(
365
+ `line:${file.path}`,
366
+ `[omp-opsx-addon] concurrency: dropping unterminated log line > ${MAX_LOG_LINE_BYTES}B in ${file.path}`,
367
+ );
368
+ tracked.offset = file.size;
369
+ }
370
+ return [];
371
+ }
372
+ const consumed = text.slice(0, lastNewline + 1);
373
+ tracked.offset += Buffer.byteLength(consumed, 'utf8');
374
+ return consumed.split('\n').slice(0, -1);
375
+ }
376
+
377
+ private warnOnce(key: string, message: string): void {
378
+ if (this.warned.has(key)) return;
379
+ this.warned.add(key);
380
+ this.warn(message);
381
+ }
382
+ }
383
+
384
+ function describe(err: unknown): string {
385
+ return err instanceof Error ? err.message : String(err);
386
+ }
387
+
388
+ /** Cheap epoch-ms extraction for buffer retention (`fallback` when unusable). */
389
+ function lineTimestamp(raw: string, fallback: number): number {
390
+ try {
391
+ const parsed: unknown = JSON.parse(raw);
392
+ if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
393
+ const ts = (parsed as Record<string, unknown>).timestamp;
394
+ if (typeof ts === 'number' && Number.isFinite(ts)) return ts;
395
+ if (typeof ts === 'string') {
396
+ const ms = Date.parse(ts);
397
+ if (Number.isFinite(ms)) return ms;
398
+ }
399
+ }
400
+ } catch {
401
+ // not JSON — retention falls back to the tick clock
402
+ }
403
+ return fallback;
404
+ }
405
+
406
+ // ── ① explicit failure classification (task 3.4) ────────────────────────────
407
+
408
+ /** Classes of signal ①: an explicit provider-level failure. */
409
+ export type ExplicitFailureKind = 'rate-limit' | 'timeout';
410
+
411
+ const TIMEOUT_OR_5XX_RE = /\btimed?\s*out\b|\btimeout\b|\bETIMEDOUT\b|\bECONNRESET\b|\bhttp[^\d]{0,3}5\d\d\b|\b5\d\d\b/;
412
+
413
+ /**
414
+ * Classify an error/response text as signal ①. `rate-limit` reuses the existing
415
+ * 429 diagnosis (`hasRateLimitError`); 5xx / timeouts fold into `timeout`.
416
+ * Returns `null` for anything that is not a provider-level failure.
417
+ */
418
+ export function classifyExplicitFailure(text: string): ExplicitFailureKind | null {
419
+ if (hasRateLimitError(text)) return 'rate-limit';
420
+ if (TIMEOUT_OR_5XX_RE.test(text)) return 'timeout';
421
+ return null;
422
+ }
423
+
424
+ interface FailureCounts {
425
+ rateLimit: number;
426
+ timeout: number;
427
+ }
428
+
429
+ function emptyCounts(): FailureCounts {
430
+ return { rateLimit: 0, timeout: 0 };
431
+ }
432
+
433
+ // ── Control loop ────────────────────────────────────────────────────────────
434
+
435
+ /**
436
+ * Durable per-provider state for a starting loop. A missing file is the normal
437
+ * first-run path (silent); an unreadable one degrades to `{}` plus a warn, i.e.
438
+ * to the shared config value — never to「无限制」.
439
+ */
440
+ function loadPersistedState(statePath: string, warn: ConcurrencyWarn): ConcurrencyStateFile {
441
+ try {
442
+ return loadConcurrencyState({ statePath, warn });
443
+ } catch (err) {
444
+ warn(
445
+ `[omp-opsx-addon] concurrency: state file ${statePath} read failed (${describe(err)}); falling back to the shared config value`,
446
+ );
447
+ return {};
448
+ }
449
+ }
450
+
451
+ export interface ConcurrencyControlOptions {
452
+ /** Config root `config.yml` the writer reads/merges/writes. */
453
+ configPath: string;
454
+ /** Managed providers (already resolved: `concurrency_providers` ∪ limits keys). */
455
+ managedProviders: readonly string[];
456
+ /** Live tuner knobs; read on every tick so `eval_interval` changes take effect. */
457
+ tuner: TunerOptions;
458
+ /** Injected clock (epoch ms). */
459
+ now: () => number;
460
+ /** Filesystem seam for the log reader. */
461
+ fs: ConcurrencyFsPort;
462
+ /** Lease owner id (`pid` + start time); used for the lease and the summary. */
463
+ owner: string;
464
+ /** Agent dir; log dir / DB path are derived from it when not given. */
465
+ agentDir?: string;
466
+ /** Explicit log directory (overrides the derived one). */
467
+ logDir?: string;
468
+ /** `model_perf` source. `null` disables the source; omitted → sqlite port. */
469
+ modelPerf?: ModelPerfPort | null;
470
+ /** `agent.db` path when the sqlite port is derived (default `<agentDir>/agent.db`). */
471
+ dbPath?: string;
472
+ /** Cross-process ① aggregation channel; `null`/omitted disables it (warn + local only). */
473
+ redis?: ConcurrencyRedisPort | null;
474
+ /** Settings hot-apply seam; omitted → file write only. */
475
+ settings?: SettingsPort | null;
476
+ /** Per-provider ceiling overrides (`concurrency_ceiling_by_provider`). */
477
+ ceilingByProvider?: Record<string, number>;
478
+ /** Warn sink. */
479
+ warn?: ConcurrencyWarn;
480
+ /** Lease TTL (`concurrency_lease_ttl_ms`). */
481
+ leaseTtlMs?: number;
482
+ /** Minimum spacing between two successful writes (`concurrency_write_min_interval_ms`). */
483
+ writeMinIntervalMs?: number;
484
+ /** File-lease path (default `<configRoot>/opsx-concurrency-tuner.lock`). */
485
+ lockPath?: string;
486
+ /** Writer lock path (default `<configPath>.lock`, per the existing protocol). */
487
+ writeLockPath?: string;
488
+ /** Newest log files read per tick. */
489
+ maxLogFiles?: number;
490
+ /** Lease factory seam (default: the writer's `createLease`). */
491
+ leaseFactory?: ConcurrencyLeaseFactory;
492
+ }
493
+
494
+ export interface ProviderTickView {
495
+ provider: string;
496
+ limit: number;
497
+ badKind: BadEventKind | null;
498
+ decreased: boolean;
499
+ increased: boolean;
500
+ queueSuppressed: boolean;
501
+ cooldownActive: boolean;
502
+ /** Signal ① count for this window (own + aggregated). */
503
+ rateLimitCount: number;
504
+ timeoutCount: number;
505
+ queueWaitP50Ms: number | null;
506
+ queueWaitP90Ms: number | null;
507
+ /** Median generation duration of this window's samples (`null` = no samples). */
508
+ latencyMedianMs: number | null;
509
+ /**
510
+ * The throughput baseline *after* this window (`null` = no usable window yet).
511
+ * Only a healthy, non-queue-masked window may move it (same verdict as the
512
+ * latency baseline); exposed so that rule is observable end-to-end.
513
+ */
514
+ tokensPerMsEwma: number | null;
515
+ written: boolean;
516
+ }
517
+
518
+ export interface ConcurrencyTickResult {
519
+ /** `false` when the absolute-time gate skipped this tick. */
520
+ evaluated: boolean;
521
+ providers: Record<string, ProviderTickView>;
522
+ }
523
+
524
+ /**
525
+ * Signal collector + control loop.
526
+ *
527
+ * {@link onTick} is the only scheduled entry point and it is driven by the
528
+ * host's existing render tick: the first call arms the absolute-time gate, and
529
+ * every later call evaluates at most once per `concurrency_eval_interval_ms`.
530
+ * The loop calls the pure {@link evaluateWindow} state machine and the writer's
531
+ * config/lease API; it owns no timer and no IO other than the injected ports.
532
+ */
533
+ export class ConcurrencyController {
534
+ private readonly fs: ConcurrencyFsPort;
535
+ /** Config-root `opsx-concurrency-state.json` (design D7). */
536
+ private readonly statePath: string;
537
+ private readonly tuner: TunerOptions;
538
+ private readonly now: () => number;
539
+ private readonly warn: ConcurrencyWarn;
540
+ private readonly logSource: IncrementalLogSource;
541
+ private readonly modelPerf: ModelPerfPort | null;
542
+ private readonly redis: ConcurrencyRedisPort | null;
543
+ private readonly settings: SettingsPort | null;
544
+ private readonly leaseFactory: ConcurrencyLeaseFactory;
545
+ private readonly leaseTtlMs: number;
546
+ private readonly writeMinIntervalMs: number;
547
+ private readonly lockPath: string;
548
+ private readonly writeLockPath: string | undefined;
549
+ private readonly redisPrefix: string;
550
+
551
+ /** Managed providers, canonicalized, first-seen order preserved. */
552
+ readonly providers: readonly string[];
553
+ readonly owner: string;
554
+ readonly configPath: string;
555
+
556
+ private readonly states = new Map<string, ProviderConcurrencyState>();
557
+ /**
558
+ * Body handed to the last persist step. `toStateFile` uses it to keep an
559
+ * unchanged entry's `updatedAt`, so a no-op window serializes byte-identical
560
+ * to the file on disk and `saveConcurrencyState` skips the write (P2-2).
561
+ */
562
+ private lastStateFile: ConcurrencyStateFile | null = null;
563
+ private readonly baselines = new Map<string, PerfBaseline>();
564
+ private readonly lastWriteAt = new Map<string, number>();
565
+ private readonly lastWindowRateLimit = new Map<string, number>();
566
+ private readonly lastWindowTimeout = new Map<string, number>();
567
+ private readonly ceilingByProvider: Record<string, number>;
568
+
569
+ private localEvents = new Map<string, FailureCounts>();
570
+ private logBuffer: Array<{ at: number; raw: string }> = [];
571
+ private queueSamples: QueueWaitSample[] = [];
572
+ private previousPerfRows: ModelPerfRow[] = [];
573
+ private perfSeeded = false;
574
+ private lease: ConcurrencyLease | null = null;
575
+ private wasLeaseHolder = false;
576
+ private lastEvalAt: number | null = null;
577
+ private selfChecked = false;
578
+ private logSourceAvailable = true;
579
+ private aggregationAvailable = true;
580
+
581
+ constructor(options: ConcurrencyControlOptions) {
582
+ this.configPath = options.configPath;
583
+ this.owner = options.owner;
584
+ this.fs = options.fs;
585
+ this.tuner = options.tuner;
586
+ this.now = options.now;
587
+ this.warn = options.warn ?? (() => {});
588
+ this.ceilingByProvider = options.ceilingByProvider ?? {};
589
+ this.leaseTtlMs = options.leaseTtlMs ?? TUNER_DEFAULTS.evalIntervalMs * 2;
590
+ const agentDir = options.agentDir ?? null;
591
+ this.lockPath = options.lockPath ?? join(dirname(resolvePath(options.configPath)), LEASE_LOCK_FILENAME);
592
+ this.writeLockPath = options.writeLockPath;
593
+ this.redisPrefix = `${RATE_LIMIT_REDIS_PREFIX}:${configRootHash(options.configPath)}`;
594
+ this.leaseFactory = options.leaseFactory ?? defaultLeaseFactory;
595
+ this.writeMinIntervalMs = options.writeMinIntervalMs ?? DEFAULT_WRITE_MIN_INTERVAL_MS;
596
+ this.modelPerf = options.modelPerf === undefined
597
+ ? createSqliteModelPerfPort(options.dbPath ?? deriveDbPath(agentDir))
598
+ : options.modelPerf;
599
+ this.redis = options.redis ?? null;
600
+ this.settings = options.settings ?? null;
601
+ this.logSource = new IncrementalLogSource(this.fs, options.logDir ?? deriveLogDir(agentDir), {
602
+ maxFiles: options.maxLogFiles,
603
+ warn: this.warn,
604
+ });
605
+
606
+ const providers: string[] = [];
607
+ const seen = new Set<string>();
608
+ for (const id of options.managedProviders) {
609
+ const canonical = canonicalizeProvider(id);
610
+ if (!canonical || seen.has(canonical)) continue;
611
+ seen.add(canonical);
612
+ providers.push(canonical);
613
+ }
614
+ this.providers = providers;
615
+
616
+ // Seed from the config root before the first tick: the persisted learned
617
+ // limit wins, then the shared config value, then `initial` (all clamped).
618
+ // Doing it here — not lazily inside `evaluate` — keeps `stateFor` and
619
+ // `summaryLines` truthful from construction on (design D7).
620
+ this.statePath = join(dirname(resolvePath(options.configPath)), STATE_FILE_NAME);
621
+ this.seedStates(loadPersistedState(this.statePath, this.warn));
622
+ }
623
+
624
+ /**
625
+ * Advance the loop from the host's existing render tick. Absolute-time
626
+ * gated off the last evaluation: the first call arms the gate without
627
+ * evaluating (a zero-length window must not count as clean); later calls
628
+ * evaluate at most once per `tuner.evalIntervalMs`, so a changed interval
629
+ * takes effect on the next tick.
630
+ */
631
+ async onTick(): Promise<ConcurrencyTickResult> {
632
+ const now = this.now();
633
+ if (!this.selfChecked) {
634
+ this.selfChecked = true;
635
+ this.runLogSelfCheck();
636
+ }
637
+ const interval = Number.isFinite(this.tuner.evalIntervalMs) && this.tuner.evalIntervalMs > 0
638
+ ? this.tuner.evalIntervalMs
639
+ : TUNER_DEFAULTS.evalIntervalMs;
640
+ if (this.lastEvalAt === null) {
641
+ // Arm the gate and snapshot the log tail positions: the first
642
+ // window starts here, so its log bytes are read, not skipped.
643
+ this.lastEvalAt = now;
644
+ this.logSource.readNewLines();
645
+ return { evaluated: false, providers: {} };
646
+ }
647
+ const elapsed = now - this.lastEvalAt;
648
+ if (elapsed < interval) return { evaluated: false, providers: {} };
649
+ this.lastEvalAt = now;
650
+ const result = await this.evaluate(now, elapsed);
651
+ // Fire-and-forget persistence (task 5.1). It sits on the evaluated
652
+ // branch, so a busy render tick cannot rewrite the state file every
653
+ // second, and it is never awaited: the tick MUST NOT block on the disk.
654
+ // Writer-gated for two reasons: a non-writer's `L` is only a local
655
+ // mirror of the shared gate (persisting it would let a restart resume a
656
+ // stale prediction and have two processes fight over one file), and a
657
+ // read-only-degraded loop MUST NOT write anything at all
658
+ // (spec「租约机制整体不可用时只读降级」).
659
+ if (this.isWriter()) {
660
+ const state = toStateFile(this.states, now, this.lastStateFile);
661
+ this.lastStateFile = state;
662
+ void saveConcurrencyState({
663
+ statePath: this.statePath,
664
+ state,
665
+ warn: this.warn,
666
+ });
667
+ }
668
+ return result;
669
+ }
670
+
671
+ /** Record an explicit provider-level failure observed in-process (task 3.4). */
672
+ noteExplicitFailure(provider: string, text: string): boolean {
673
+ const kind = classifyExplicitFailure(text);
674
+ if (kind === null) return false;
675
+ const canonical = canonicalizeProvider(provider);
676
+ if (!canonical) return false;
677
+ const counts = this.localEvents.get(canonical) ?? emptyCounts();
678
+ if (kind === 'rate-limit') counts.rateLimit += 1;
679
+ else counts.timeout += 1;
680
+ this.localEvents.set(canonical, counts);
681
+ return true;
682
+ }
683
+
684
+ /** Whether the debug-level in-flight log signal (②) passed the runtime self-check. */
685
+ logSignalAvailable(): boolean {
686
+ return this.logSourceAvailable;
687
+ }
688
+
689
+ /** Whether cross-process ① aggregation through Redis is usable. */
690
+ aggregationUsable(): boolean {
691
+ return this.aggregationAvailable && this.redis !== null;
692
+ }
693
+
694
+ /** Current lease mode (`null` before the first evaluation). */
695
+ leaseMode(): LeaseMode | null {
696
+ return this.lease?.mode ?? null;
697
+ }
698
+
699
+ /** Whether this process may write to the shared config right now. */
700
+ private isWriter(): boolean {
701
+ return this.lease?.held === true && this.lease.mode !== 'readonly';
702
+ }
703
+
704
+ /**
705
+ * Lease holder as last observed: this process while it holds the lease, the
706
+ * peer's owner id when a peer holds it, `null` when nobody does. Lets a
707
+ * non-writer's summary name the holder instead of only denying being one.
708
+ */
709
+ leaseOwner(): string | null {
710
+ return this.lease?.held === true ? this.owner : this.lease?.holderId ?? null;
711
+ }
712
+
713
+ /** Current state of a provider (test/inspection seam). */
714
+ stateFor(provider: string): ProviderConcurrencyState | null {
715
+ return this.states.get(canonicalizeProvider(provider)) ?? null;
716
+ }
717
+
718
+ /** Summary lines for the `/pick-model` section (task 2.7 format). */
719
+ summaryLines(): string[] {
720
+ const now = this.now();
721
+ const views = this.providers.map((provider) => {
722
+ const state = this.states.get(provider) ?? this.stateSeed(provider, null);
723
+ const { p50, p90 } = queueWaitPercentiles(
724
+ this.queueSamples.filter((s) => s.provider === provider),
725
+ now,
726
+ this.tuner.queueWaitWindowMs,
727
+ );
728
+ return {
729
+ provider,
730
+ limit: state.limit,
731
+ floor: state.floor,
732
+ ceiling: state.ceiling,
733
+ rateLimitCount: this.lastWindowRateLimit.get(provider) ?? 0,
734
+ timeoutCount: this.lastWindowTimeout.get(provider) ?? 0,
735
+ queueWaitP50Ms: p50,
736
+ queueWaitP90Ms: p90,
737
+ lastWriteAt: this.lastWriteAt.get(provider) ?? null,
738
+ };
739
+ });
740
+ const lines: string[] = [];
741
+ if (!this.logSourceAvailable) {
742
+ lines.push('并发控制降级: 日志 debug 信号不可用(未探测到 blocked/wait completed 行),排队等待护栏已失效');
743
+ }
744
+ if (!this.aggregationAvailable || this.redis === null) {
745
+ lines.push('并发控制降级: Redis 汇聚不可用,限流信号仅本进程可见(以信号②③补偿)');
746
+ }
747
+ lines.push(...formatConcurrencySummary({
748
+ providers: views,
749
+ leaseOwner: this.leaseOwner(),
750
+ selfId: this.owner,
751
+ }));
752
+ return lines;
753
+ }
754
+
755
+ /**
756
+ * Release the lease (tuner stop / plugin unload) and run the shutdown
757
+ * fail-safe. Never deletes config keys and never writes a managed provider
758
+ * back to unlimited — the last finite value stays in the shared config.
759
+ */
760
+ async shutdown(): Promise<void> {
761
+ const lease = this.lease;
762
+ this.lease = null;
763
+ this.wasLeaseHolder = false;
764
+ if (lease) {
765
+ try {
766
+ await lease.release();
767
+ } catch (err) {
768
+ this.warn(`[omp-opsx-addon] concurrency: lease release failed: ${describe(err)}`);
769
+ }
770
+ }
771
+ await stopFailSafe({ statePath: this.statePath, warn: this.warn });
772
+ }
773
+
774
+ // ── evaluation ──────────────────────────────────────────────────────────
775
+
776
+ /**
777
+ * One evaluation window. `elapsedMs` is the observed gap since the previous
778
+ * evaluation: `concurrency_up_ms` measures clean time against it, while the
779
+ * gate in {@link onTick} owns the cadence.
780
+ */
781
+ private async evaluate(now: number, elapsedMs: number): Promise<ConcurrencyTickResult> {
782
+ const result: ConcurrencyTickResult = { evaluated: true, providers: {} };
783
+ const sharedMap = this.readSharedLimits();
784
+
785
+ await this.ensureLease(now);
786
+ const isWriter = this.isWriter();
787
+
788
+ if (isWriter && !this.wasLeaseHolder) {
789
+ this.alignToSharedValues(sharedMap);
790
+ }
791
+ this.wasLeaseHolder = isWriter;
792
+
793
+ this.collectLogSignals(now);
794
+ const perfWindows = this.collectPerfWindows();
795
+ const failures = await this.collectFailures(isWriter);
796
+
797
+ for (const provider of this.providers) {
798
+ const state = this.states.get(provider) ?? this.seedState(provider, sharedMap[provider] ?? null, null);
799
+ this.states.set(provider, state);
800
+ const perf = perfWindows.get(provider) ?? null;
801
+ const baseline = this.baselines.get(provider) ?? null;
802
+ const failure = failures.get(provider) ?? emptyCounts();
803
+ const genSamplesMs = perf ? genSamplesFromPerfWindow(perf, this.tuner.minSamples) : [];
804
+ // Signal ② is derived once per provider and then handed in explicitly:
805
+ // the evaluator's mask/guardrail and this window's reported p50/p90 are
806
+ // the same number, not two derivations of the same samples.
807
+ const queueWait = this.queueSamples.filter((s) => s.provider === provider);
808
+ const { p50, p90 } = queueWaitPercentiles(queueWait, now, this.tuner.queueWaitWindowMs);
809
+ const signals: WindowSignals = {
810
+ // R1 evidence: the window really carried generation samples (the
811
+ // `model_perf` delta is the only source — no synthetic filler), so
812
+ // an idle provider freezes instead of climbing.
813
+ observed: genSamplesMs.length > 0,
814
+ rateLimitCount: failure.rateLimit + failure.timeout,
815
+ genSamplesMs,
816
+ tokensPerMs: perf ? perf.tokensPerMs : null,
817
+ tokensPerMsEwma: baseline?.tokensPerMs ?? null,
818
+ queueWait,
819
+ queueWaitP50Ms: p50,
820
+ };
821
+ const evaluation = evaluateWindow(state, signals, now, this.tuner, elapsedMs);
822
+ // Decay (task 5.3): a provider that has gone
823
+ // `concurrency_decay_after_ms` without an observation moves one
824
+ // notch toward `initial` — and the move goes out through the very
825
+ // same gate/lease/writer path below, never a second write path.
826
+ const decay = applyDecayStep(evaluation.state, {
827
+ now,
828
+ decayAfterMs: this.tuner.decayAfterMs,
829
+ initial: evaluation.state.initial,
830
+ floor: evaluation.state.floor,
831
+ });
832
+ const next = decay.state;
833
+ this.states.set(provider, next);
834
+ // Baseline hygiene, on the very verdict the evaluator reached: a
835
+ // queue-masked or bad window MUST NOT move the throughput baseline
836
+ // either — else the load-driven dip becomes the new normal and blinds
837
+ // the fallback that is supposed to catch it.
838
+ if (perf && evaluation.baselineEligible) {
839
+ this.baselines.set(provider, updatePerfBaseline(baseline, perf, this.tuner));
840
+ }
841
+ this.lastWindowRateLimit.set(provider, failure.rateLimit);
842
+ this.lastWindowTimeout.set(provider, failure.timeout);
843
+
844
+ const view: ProviderTickView = {
845
+ provider,
846
+ limit: next.limit,
847
+ badKind: evaluation.badKind,
848
+ decreased: evaluation.decreased,
849
+ increased: evaluation.increased,
850
+ queueSuppressed: evaluation.queueSuppressed,
851
+ cooldownActive: evaluation.cooldownActive,
852
+ rateLimitCount: failure.rateLimit,
853
+ timeoutCount: failure.timeout,
854
+ queueWaitP50Ms: p50,
855
+ queueWaitP90Ms: p90,
856
+ latencyMedianMs: evaluation.latencyMedianMs,
857
+ tokensPerMsEwma: this.baselines.get(provider)?.tokensPerMs ?? null,
858
+ written: false,
859
+ };
860
+ // Write candidates: a move this window (up / down / decay), or a
861
+ // divergence from the shared value that no window can repair on its
862
+ // own — the first-window catch-up and a degenerate range
863
+ // (`floor == ceiling`, e.g. a per-provider ceiling override below the
864
+ // static value, where neither increase nor decrease can ever fire and
865
+ // the host gate would keep serving the stale value we display).
866
+ // `persistLimit` still applies the |Δ| >= 1 gate, the write interval
867
+ // and the lease — no second write path.
868
+ const shared = sharedMap[provider];
869
+ const divergedFromShared = typeof shared === 'number' && shared !== next.limit;
870
+ if ((evaluation.increased || evaluation.decreased || decay.moved || divergedFromShared) && isWriter) {
871
+ view.written = this.persistLimit(provider, next.limit, sharedMap, now);
872
+ }
873
+ result.providers[provider] = view;
874
+ }
875
+ return result;
876
+ }
877
+
878
+ private stateSeed(provider: string, sharedLimit: number | null): ProviderConcurrencyState {
879
+ return createProviderState({
880
+ sharedLimit,
881
+ start: this.tuner.start,
882
+ ceilingOverride: this.ceilingByProvider[provider] ?? null,
883
+ options: this.tuner,
884
+ });
885
+ }
886
+
887
+ /** Seed `states` for every managed provider from the durable file / shared config. */
888
+ private seedStates(persisted: ConcurrencyStateFile): void {
889
+ const shared = this.readSharedLimits();
890
+ for (const provider of this.providers) {
891
+ this.states.set(provider, this.seedState(provider, shared[provider] ?? null, persisted[provider] ?? null));
892
+ }
893
+ }
894
+
895
+ /**
896
+ * One provider's starting state (design D7). Without a durable entry this
897
+ * is just the seed value (`shared ?? concurrency_start`, clamped). With one,
898
+ * the learned limit and its trajectory resume:
899
+ *
900
+ * - `limit` — {@link restoreLimit} resolves persisted → shared → `initial`.
901
+ * - `initial` — the *persisted* decay target, not a re-derivation from the
902
+ * current shared value: the controller writes `L` back into the shared
903
+ * config, so re-deriving would make the learned limit its own decay target
904
+ * and permanently disable decay after the first restart.
905
+ * - `lastObservationAt` — the persisted anchor, so decay resumes on the real
906
+ * idle clock. A state that never observed anything anchors at the process
907
+ * start: anchoring at 0 would make every window look「idle for an epoch」
908
+ * and decay a freshly raised `L` straight back down.
909
+ */
910
+ private seedState(
911
+ provider: string,
912
+ sharedLimit: number | null,
913
+ persisted: PersistedProviderState | null,
914
+ ): ProviderConcurrencyState {
915
+ const seed = this.stateSeed(provider, sharedLimit);
916
+ if (!persisted) return { ...seed, lastObservationAt: this.now() };
917
+ return {
918
+ ...seed,
919
+ limit: restoreLimit(persisted, {
920
+ floor: seed.floor,
921
+ ceiling: seed.ceiling,
922
+ sharedValue: sharedLimit,
923
+ initial: seed.initial,
924
+ }),
925
+ initial: clampLimit(persisted.initial, seed.floor, seed.ceiling),
926
+ badStreak: persisted.badStreak,
927
+ cleanWindows: persisted.cleanStreak,
928
+ ewmaGenMs: persisted.ewmaGenMs ?? seed.ewmaGenMs,
929
+ lastObservationAt: persisted.lastObservationAt > 0 ? persisted.lastObservationAt : this.now(),
930
+ };
931
+ }
932
+
933
+ /**
934
+ * Re-seed every provider's `L` from the shared config value when this
935
+ * process first becomes the writer (spec: takeover must not fall back to a
936
+ * local `initial`/`concurrency_start`, and the alignment itself writes
937
+ * nothing).
938
+ */
939
+ private alignToSharedValues(sharedMap: Record<string, number>): void {
940
+ for (const provider of this.providers) {
941
+ const state = this.states.get(provider);
942
+ if (!state) continue;
943
+ state.limit = resolveTakeoverStart(sharedMap[provider] ?? null, {
944
+ floor: state.floor,
945
+ ceiling: state.ceiling,
946
+ start: state.initial,
947
+ });
948
+ }
949
+ }
950
+
951
+ private readSharedLimits(): Record<string, number> {
952
+ const map: Record<string, number> = {};
953
+ try {
954
+ for (const row of readManagedLimits(this.configPath, this.providers)) {
955
+ const canonical = canonicalizeProvider(row.provider);
956
+ if (canonical && typeof row.value === 'number' && Number.isFinite(row.value)) {
957
+ map[canonical] = Math.trunc(row.value);
958
+ }
959
+ }
960
+ } catch (err) {
961
+ this.warn(`[omp-opsx-addon] concurrency: reading providers.maxInFlightRequests failed: ${describe(err)}`);
962
+ }
963
+ return map;
964
+ }
965
+
966
+ private async ensureLease(now: number): Promise<void> {
967
+ if (this.lease) {
968
+ if (this.lease.mode === 'readonly') return;
969
+ let renewed = false;
970
+ try {
971
+ renewed = await this.lease.renew();
972
+ } catch (err) {
973
+ this.warn(`[omp-opsx-addon] concurrency: lease renew failed: ${describe(err)}`);
974
+ }
975
+ this.lease.held = renewed;
976
+ if (!renewed) {
977
+ // Lost the lease: drop the handle so the next tick re-acquires it
978
+ // (renewing a lease someone else owns can never succeed).
979
+ const lost = this.lease;
980
+ this.lease = null;
981
+ try {
982
+ await lost.release();
983
+ } catch (err) {
984
+ this.warn(`[omp-opsx-addon] concurrency: lease release after loss failed: ${describe(err)}`);
985
+ }
986
+ }
987
+ return;
988
+ }
989
+ try {
990
+ this.lease = await this.leaseFactory({
991
+ redis: this.redis,
992
+ lockPath: this.lockPath,
993
+ owner: this.owner,
994
+ ttlMs: this.leaseTtlMs,
995
+ now: this.now,
996
+ });
997
+ if (this.lease.mode === 'readonly') {
998
+ this.warn('[omp-opsx-addon] concurrency: lease unavailable (Redis + file lease both failed); running read-only');
999
+ }
1000
+ } catch (err) {
1001
+ this.warn(`[omp-opsx-addon] concurrency: lease acquisition failed: ${describe(err)}`);
1002
+ this.lease = { held: false, mode: 'readonly', holderId: null, renew: async () => false, release: async () => {} };
1003
+ }
1004
+ }
1005
+
1006
+ /**
1007
+ * Signal ②: read the new bytes of every log file and re-pair the retained
1008
+ * recent lines. Pairing is redone over the whole recent buffer because a
1009
+ * `blocked` line and its `wait completed` line routinely land in different
1010
+ * ticks; recomputing (instead of appending) also keeps them from being
1011
+ * counted twice. Lines older than two guardrail windows are dropped.
1012
+ */
1013
+ private collectLogSignals(now: number): void {
1014
+ let lines: string[];
1015
+ try {
1016
+ lines = this.logSource.readNewLines();
1017
+ } catch (err) {
1018
+ this.warn(`[omp-opsx-addon] concurrency: log collection failed: ${describe(err)}`);
1019
+ return;
1020
+ }
1021
+ if (lines.length === 0 && this.logBuffer.length === 0) return;
1022
+ for (const raw of lines) this.logBuffer.push({ at: lineTimestamp(raw, now), raw });
1023
+ const cutoff = now - 2 * this.tuner.queueWaitWindowMs;
1024
+ this.logBuffer = this.logBuffer.filter((entry) => entry.at >= cutoff);
1025
+ if (this.logBuffer.length > MAX_BUFFERED_LOG_LINES) {
1026
+ this.logBuffer = this.logBuffer.slice(-MAX_BUFFERED_LOG_LINES);
1027
+ }
1028
+ const parsed = parseInFlightLogLines(this.logBuffer.map((entry) => entry.raw));
1029
+ this.queueSamples = parsed.samples.map((sample) => ({
1030
+ ...sample,
1031
+ provider: canonicalizeProvider(sample.provider),
1032
+ }));
1033
+ }
1034
+
1035
+ private collectPerfWindows(): Map<string, ProviderPerfWindow> {
1036
+ const windows = new Map<string, ProviderPerfWindow>();
1037
+ if (!this.modelPerf) return windows;
1038
+ let rows: ModelPerfRow[];
1039
+ try {
1040
+ rows = this.modelPerf.readModelPerf();
1041
+ } catch (err) {
1042
+ this.warn(`[omp-opsx-addon] concurrency: model_perf read failed, skipping source: ${describe(err)}`);
1043
+ return windows;
1044
+ }
1045
+ if (!this.perfSeeded) {
1046
+ // The table holds cumulative counters: the first read is the window
1047
+ // baseline, not a window's worth of history.
1048
+ this.perfSeeded = true;
1049
+ this.previousPerfRows = rows;
1050
+ return windows;
1051
+ }
1052
+ for (const window of diffModelPerf(this.previousPerfRows, rows, this.tuner)) {
1053
+ windows.set(canonicalizeProvider(window.provider), window);
1054
+ }
1055
+ this.previousPerfRows = rows;
1056
+ return windows;
1057
+ }
1058
+
1059
+ /**
1060
+ * Signal ① for this window. Non-writers report their local counts into the
1061
+ * per-provider Redis counters (atomically drained by the writer with
1062
+ * `GETSET`), so a 429 observed only on a non-writer's LLM path still drives
1063
+ * the shared down-scale. Redis unavailable → warn once and use local counts
1064
+ * (spec: evaluation must continue on signals ②③).
1065
+ */
1066
+ private async collectFailures(isWriter: boolean): Promise<Map<string, FailureCounts>> {
1067
+ const local = this.localEvents;
1068
+ this.localEvents = new Map();
1069
+ const source = new Map<string, FailureCounts>();
1070
+ for (const [provider, counts] of local) source.set(provider, { ...counts });
1071
+
1072
+ if (this.redis === null) {
1073
+ if (this.aggregationAvailable) {
1074
+ this.aggregationAvailable = false;
1075
+ this.warn('[omp-opsx-addon] concurrency: Redis aggregation unavailable, continuing on signals ②③');
1076
+ }
1077
+ return source;
1078
+ }
1079
+ try {
1080
+ if (!isWriter) {
1081
+ await this.reportLocalFailures(local);
1082
+ return source;
1083
+ }
1084
+ const remote = await this.drainRemoteFailures();
1085
+ for (const [provider, counts] of remote) {
1086
+ const merged = source.get(provider) ?? emptyCounts();
1087
+ merged.rateLimit += counts.rateLimit;
1088
+ merged.timeout += counts.timeout;
1089
+ source.set(provider, merged);
1090
+ }
1091
+ return source;
1092
+ } catch (err) {
1093
+ if (this.aggregationAvailable) {
1094
+ this.aggregationAvailable = false;
1095
+ this.warn(`[omp-opsx-addon] concurrency: Redis aggregation unavailable, continuing on signals ②③: ${describe(err)}`);
1096
+ }
1097
+ return source;
1098
+ }
1099
+ }
1100
+
1101
+ private async reportLocalFailures(local: Map<string, FailureCounts>): Promise<void> {
1102
+ const commands: string[][] = [];
1103
+ for (const [provider, counts] of local) {
1104
+ if (counts.rateLimit > 0) {
1105
+ commands.push(['INCRBY', this.failureKey(provider, 'rl'), String(counts.rateLimit)]);
1106
+ commands.push(['EXPIRE', this.failureKey(provider, 'rl'), String(this.aggregationTtlSeconds())]);
1107
+ }
1108
+ if (counts.timeout > 0) {
1109
+ commands.push(['INCRBY', this.failureKey(provider, 'to'), String(counts.timeout)]);
1110
+ commands.push(['EXPIRE', this.failureKey(provider, 'to'), String(this.aggregationTtlSeconds())]);
1111
+ }
1112
+ }
1113
+ if (commands.length === 0) return;
1114
+ await this.redis!.pipeline(commands);
1115
+ }
1116
+
1117
+ private async drainRemoteFailures(): Promise<Map<string, FailureCounts>> {
1118
+ const commands: string[][] = [];
1119
+ for (const provider of this.providers) {
1120
+ for (const field of ['rl', 'to'] as const) {
1121
+ commands.push(['GETSET', this.failureKey(provider, field), '0']);
1122
+ commands.push(['EXPIRE', this.failureKey(provider, field), String(this.aggregationTtlSeconds())]);
1123
+ }
1124
+ }
1125
+ if (commands.length === 0) return new Map();
1126
+ const replies = await this.redis!.pipeline(commands);
1127
+ const out = new Map<string, FailureCounts>();
1128
+ let index = 0;
1129
+ for (const provider of this.providers) {
1130
+ const counts = { rateLimit: readCounter(replies[index]), timeout: readCounter(replies[index + 2]) };
1131
+ index += 4;
1132
+ if (counts.rateLimit > 0 || counts.timeout > 0) out.set(provider, counts);
1133
+ }
1134
+ return out;
1135
+ }
1136
+
1137
+ private failureKey(provider: string, field: 'rl' | 'to'): string {
1138
+ return `${this.redisPrefix}:${provider}:${field}`;
1139
+ }
1140
+
1141
+ private aggregationTtlSeconds(): number {
1142
+ const interval = Number.isFinite(this.tuner.evalIntervalMs) && this.tuner.evalIntervalMs > 0
1143
+ ? this.tuner.evalIntervalMs
1144
+ : TUNER_DEFAULTS.evalIntervalMs;
1145
+ return Math.max(1, Math.ceil((interval * AGGREGATION_TTL_INTERVALS) / 1000));
1146
+ }
1147
+
1148
+ private persistLimit(
1149
+ provider: string,
1150
+ limit: number,
1151
+ sharedMap: Record<string, number>,
1152
+ now: number,
1153
+ ): boolean {
1154
+ // Write gate (task 4.2): |Δ| >= 1 AND more than
1155
+ // concurrency_write_min_interval_ms since this provider's last write.
1156
+ const gate = shouldWriteLimit({
1157
+ newValue: limit,
1158
+ oldValue: sharedMap[provider] ?? null,
1159
+ lastWriteAt: this.lastWriteAt.get(provider) ?? null,
1160
+ now,
1161
+ minIntervalMs: this.writeMinIntervalMs,
1162
+ });
1163
+ if (!gate.write) return false;
1164
+
1165
+ const merged = buildMergedMap(sharedMap, provider, limit);
1166
+ let outcome: WriteManagedLimitResult;
1167
+ try {
1168
+ outcome = this.writeLockPath === undefined
1169
+ ? writeManagedLimit({ configPath: this.configPath, provider, merged, now })
1170
+ : writeManagedLimit({ configPath: this.configPath, provider, merged, lockPath: this.writeLockPath, now });
1171
+ } catch (err) {
1172
+ this.warn(`[omp-opsx-addon] concurrency: writing providers.maxInFlightRequests failed (keeping ${limit}): ${describe(err)}`);
1173
+ return false;
1174
+ }
1175
+ if (!outcome.written) {
1176
+ if (outcome.reason !== undefined && outcome.reason !== 'unchanged') {
1177
+ this.warn(`[omp-opsx-addon] concurrency: ${provider} limit write skipped (${outcome.reason})`);
1178
+ }
1179
+ return false;
1180
+ }
1181
+ // Adopt the in-lock read: it is the freshest view of every provider, so
1182
+ // the rest of this tick and the whole-key `settings.override` below never
1183
+ // carry a snapshot taken before another process's write landed.
1184
+ if (outcome.limits) {
1185
+ for (const [id, value] of Object.entries(outcome.limits)) sharedMap[id] = value;
1186
+ }
1187
+ sharedMap[provider] = limit;
1188
+ this.lastWriteAt.set(provider, now);
1189
+ if (this.settings) {
1190
+ try {
1191
+ const applied = applyLimit({
1192
+ settings: this.settings,
1193
+ merged: outcome.limits ?? merged,
1194
+ provider,
1195
+ value: limit,
1196
+ });
1197
+ if (!applied.applied) {
1198
+ this.warn(`[omp-opsx-addon] concurrency: settings hot-apply skipped for ${provider} (${applied.reason ?? 'unknown'})`);
1199
+ }
1200
+ } catch (err) {
1201
+ this.warn(`[omp-opsx-addon] concurrency: settings hot-apply failed for ${provider}: ${describe(err)}`);
1202
+ }
1203
+ }
1204
+ return true;
1205
+ }
1206
+
1207
+ private runLogSelfCheck(): void {
1208
+ let available: boolean;
1209
+ try {
1210
+ available = this.logSource.probeDebugInFlightLines();
1211
+ } catch (err) {
1212
+ available = false;
1213
+ this.warn(`[omp-opsx-addon] concurrency: log self-check failed: ${describe(err)}`);
1214
+ }
1215
+ this.logSourceAvailable = available;
1216
+ if (!available) {
1217
+ this.warn(
1218
+ '[omp-opsx-addon] concurrency: signal ② (queue wait) unavailable — no level=debug '
1219
+ + '"Provider in-flight limit blocked request" / "wait completed" lines in the host log; '
1220
+ + 'queue-wait guardrail disabled (signals ①③④ still active)',
1221
+ );
1222
+ }
1223
+ }
1224
+ }
1225
+
1226
+ /** Map aggregated `model_perf` deltas onto the per-request sample list `evaluateWindow` expects. */
1227
+ export function genSamplesFromPerfWindow(window: ProviderPerfWindow, minSamples: number): number[] {
1228
+ if (window.samples <= 0 || !Number.isFinite(window.genMsPerSample) || window.genMsPerSample <= 0) return [];
1229
+ const capped = Math.min(window.samples, 64);
1230
+ const count = window.samples >= minSamples ? Math.max(minSamples, capped) : window.samples;
1231
+ return new Array<number>(count).fill(window.genMsPerSample);
1232
+ }
1233
+
1234
+ function readCounter(reply: RedisReply | undefined): number {
1235
+ if (typeof reply === 'number' && Number.isFinite(reply)) return Math.max(0, Math.trunc(reply));
1236
+ if (typeof reply === 'string') {
1237
+ const value = Number.parseInt(reply, 10);
1238
+ return Number.isFinite(value) && value > 0 ? value : 0;
1239
+ }
1240
+ return 0;
1241
+ }
1242
+
1243
+ /** Convenience factory mirroring the exported class. */
1244
+ export function createConcurrencyController(options: ConcurrencyControlOptions): ConcurrencyController {
1245
+ return new ConcurrencyController(options);
1246
+ }