@genee/omp-opsx-addon 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +441 -71
- package/index.ts +1030 -208
- package/lib/agent-defs.ts +55 -82
- package/lib/complexity-router.ts +336 -0
- package/lib/concurrency-control.ts +1246 -0
- package/lib/concurrency-multiproc.worker.ts +376 -0
- package/lib/concurrency-state.ts +397 -0
- package/lib/concurrency-tuner.ts +773 -0
- package/lib/concurrency-writer.ts +894 -0
- package/lib/continuity-guard.ts +117 -0
- package/lib/direct-fetchers.ts +18 -3
- package/lib/family-filter.ts +115 -2
- package/lib/model-roles.ts +41 -39
- package/lib/model-selector.ts +300 -36
- package/lib/model-speed.ts +189 -0
- package/lib/pick-model-render.ts +414 -0
- package/lib/pipe-core.ts +325 -98
- package/lib/pipe-push.ts +136 -5
- package/lib/provider-variants.ts +154 -0
- package/lib/selection-filters.ts +42 -1
- package/lib/stall-detector.ts +185 -0
- package/lib/system-prompt.ts +17 -9
- package/lib/tiers-data.ts +40 -5
- package/lib/tiers-updater.ts +51 -8
- package/lib/unified-config.ts +732 -27
- package/lib/usage-poller.ts +44 -1
- package/lib/usage-render.ts +57 -4
- package/lib/usage-widget.ts +18 -4
- package/package.json +1 -1
- package/skills/opsx-orchestration-protocol/SKILL.md +87 -0
|
@@ -0,0 +1,894 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Concurrency writer — the writing half of `provider-concurrency-control`
|
|
3
|
+
* (spec Requirement「并发上限写入与跨进程单写者」, design D5/D6, tasks 4.1–4.9).
|
|
4
|
+
*
|
|
5
|
+
* Three responsibilities, all deterministic given their injected ports:
|
|
6
|
+
*
|
|
7
|
+
* 1. **共享现值读取** — {@link readManagedLimits} reads
|
|
8
|
+
* `providers.maxInFlightRequests` out of the config root `config.yml`
|
|
9
|
+
* (YAML) and reports the current integer per managed provider (`null` =
|
|
10
|
+
* no value on disk).
|
|
11
|
+
* 2. **门限 + 读-合并-写** — {@link shouldWriteLimit} enforces
|
|
12
|
+
* `|new − old| >= 1 && 距上次成功写入 > concurrency_write_min_interval_ms`;
|
|
13
|
+
* {@link writeManagedLimit} then does a locked read-merge-write that
|
|
14
|
+
* splices *only* the target provider's value textually, so every other
|
|
15
|
+
* key (other providers, `providers.*`, `task.*`, `retry.*`, comments,
|
|
16
|
+
* formatting) stays byte-identical, and lands the result with tmp+rename.
|
|
17
|
+
* The `<file>.lock` protocol mirrors `lib/usage-poller.ts`
|
|
18
|
+
* (`openSync(lockPath,'wx')`, stale > 5s stealable, in-lock re-read,
|
|
19
|
+
* readers lock-free).
|
|
20
|
+
* 3. **跨进程单写者** — {@link createLease} takes writer eligibility:
|
|
21
|
+
* Redis `SET key owner NX PX ttl` over the existing
|
|
22
|
+
* `RedisClient.pipeline()` command surface, degrading to a config-root
|
|
23
|
+
* file lease (`O_EXCL` atomic create + mtime expiry takeover), and finally
|
|
24
|
+
* to read-only + warn when *neither* is available (`held: false`,
|
|
25
|
+
* `mode: 'readonly'` — the caller MUST NOT write, MUST NOT fall back to
|
|
26
|
+
* unlimited).
|
|
27
|
+
* {@link resolveTakeoverStart} gives the takeover/`L` start value: the
|
|
28
|
+
* shared value clamped to `[floor, ceiling]` — never this process's
|
|
29
|
+
* `initial`/`concurrency_start`.
|
|
30
|
+
*
|
|
31
|
+
* Every IO touchpoint is an injected port (file system / redis / settings) so
|
|
32
|
+
* the module and its tests run without real IO; {@link nodeWriterFileSystem} is
|
|
33
|
+
* the production file-system adapter and the assembly layer may pass it
|
|
34
|
+
* explicitly or substitute its own.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, utimesSync, writeFileSync, writeSync } from 'fs';
|
|
38
|
+
import { dirname } from 'path';
|
|
39
|
+
import { YAML } from 'bun';
|
|
40
|
+
import { clampLimit } from './concurrency-tuner.js';
|
|
41
|
+
import { canonicalizeProvider } from './usage-resolver.js';
|
|
42
|
+
|
|
43
|
+
/** Sink for degradation notices; defaults to a no-op. */
|
|
44
|
+
export type Warn = (message: string) => void;
|
|
45
|
+
|
|
46
|
+
const NO_WARN: Warn = () => {};
|
|
47
|
+
|
|
48
|
+
/** `<file>.lock` staleness threshold — same value/protocol as `lib/usage-poller.ts`. */
|
|
49
|
+
export const WRITE_LOCK_STALE_MS = 5_000;
|
|
50
|
+
/** Lease TTL used when the caller passes a non-positive `ttlMs`. */
|
|
51
|
+
export const DEFAULT_LEASE_TTL_MS = 60_000;
|
|
52
|
+
|
|
53
|
+
const WRITE_LOCK_RETRY_MS = 10;
|
|
54
|
+
const WRITE_LOCK_ATTEMPTS = 5;
|
|
55
|
+
const FILE_LEASE_ATTEMPTS = 3;
|
|
56
|
+
|
|
57
|
+
function message(err: unknown): string {
|
|
58
|
+
return err instanceof Error ? err.message : String(err);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function codeOf(err: unknown): string | null {
|
|
62
|
+
const code = (err as { code?: unknown } | null)?.code;
|
|
63
|
+
return typeof code === 'string' ? code : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function sleepSync(ms: number): void {
|
|
67
|
+
try {
|
|
68
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
69
|
+
} catch {
|
|
70
|
+
const until = Date.now() + ms;
|
|
71
|
+
while (Date.now() < until) { /* spin */ }
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ── injectable file system port ─────────────────────────────────────
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The file-system surface this module needs. `openExclusive` MUST be an atomic
|
|
79
|
+
* exclusive create and MUST throw an error with `code === 'EEXIST'` when the
|
|
80
|
+
* path already exists — any other failure code means the file system cannot do
|
|
81
|
+
* atomic creation at all, which downgrades the lease to read-only.
|
|
82
|
+
*/
|
|
83
|
+
export interface WriterFileSystemPort {
|
|
84
|
+
/** File contents, or `null` when unreadable/absent. */
|
|
85
|
+
readFile(path: string): string | null;
|
|
86
|
+
writeFile(path: string, content: string): void;
|
|
87
|
+
rename(from: string, to: string): void;
|
|
88
|
+
/** Remove a file, ignoring absence. */
|
|
89
|
+
remove(path: string): void;
|
|
90
|
+
/** Best-effort recursive directory creation. */
|
|
91
|
+
ensureDir(dir: string): void;
|
|
92
|
+
/** `O_EXCL` create + write; throws `EEXIST` when held. */
|
|
93
|
+
openExclusive(path: string, contents: string): void;
|
|
94
|
+
/** mtime in epoch ms, or `null` when absent. */
|
|
95
|
+
statMtimeMs(path: string): number | null;
|
|
96
|
+
/** Set mtime (lease renewal). */
|
|
97
|
+
touch(path: string, mtimeMs: number): void;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Production file-system adapter (node `fs`).
|
|
102
|
+
*
|
|
103
|
+
* The mutations here are synchronous (`writeFileSync` / `renameSync` /
|
|
104
|
+
* `mkdirSync`). Known tradeoff, accepted for now: the concurrency control loop
|
|
105
|
+
* writes at most once per evaluation window (≥ 30 s apart, and only when the
|
|
106
|
+
* body actually changed), and a measured `writeFile` + `rename` of the body
|
|
107
|
+
* costs ≈0.10 ms median / ≈0.12 ms p90 (255-byte state file) and ≈0.09 ms /
|
|
108
|
+
* ≈0.14 ms (config patch) on the target machine — far below the cadence it
|
|
109
|
+
* sits on, so the render tick's fire-and-forget call is not worth an async IO
|
|
110
|
+
* pipeline (whose blast radius covers the whole writer/lease protocol).
|
|
111
|
+
* Revisit if the body grows, or the loop gains a sub-second cadence.
|
|
112
|
+
*/
|
|
113
|
+
export const nodeWriterFileSystem: WriterFileSystemPort = {
|
|
114
|
+
readFile(path) {
|
|
115
|
+
try {
|
|
116
|
+
return readFileSync(path, 'utf-8');
|
|
117
|
+
} catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
writeFile(path, content) {
|
|
122
|
+
writeFileSync(path, content, 'utf-8');
|
|
123
|
+
},
|
|
124
|
+
rename(from, to) {
|
|
125
|
+
renameSync(from, to);
|
|
126
|
+
},
|
|
127
|
+
remove(path) {
|
|
128
|
+
rmSync(path, { force: true });
|
|
129
|
+
},
|
|
130
|
+
ensureDir(dir) {
|
|
131
|
+
mkdirSync(dir, { recursive: true });
|
|
132
|
+
},
|
|
133
|
+
openExclusive(path, contents) {
|
|
134
|
+
const fd = openSync(path, 'wx');
|
|
135
|
+
try {
|
|
136
|
+
writeSync(fd, contents);
|
|
137
|
+
} catch (err) {
|
|
138
|
+
try { closeSync(fd); } catch { /* ignore */ }
|
|
139
|
+
try { rmSync(path, { force: true }); } catch { /* ignore */ }
|
|
140
|
+
throw err;
|
|
141
|
+
}
|
|
142
|
+
closeSync(fd);
|
|
143
|
+
},
|
|
144
|
+
statMtimeMs(path) {
|
|
145
|
+
try {
|
|
146
|
+
return statSync(path).mtimeMs;
|
|
147
|
+
} catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
touch(path, mtimeMs) {
|
|
152
|
+
const at = new Date(mtimeMs);
|
|
153
|
+
utimesSync(path, at, at);
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
// ── shared-value read (4.1) ─────────────────────────────────────────
|
|
158
|
+
|
|
159
|
+
/** Current shared limit for one managed provider (`null` = no value on disk). */
|
|
160
|
+
export interface ManagedLimitValue {
|
|
161
|
+
provider: string;
|
|
162
|
+
value: number | null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Canonical provider → current *integer* value found in
|
|
167
|
+
* `providers.maxInFlightRequests`. Values that are not positive integers
|
|
168
|
+
* (floats, strings, nulls) are treated as absent — they cannot serve as a
|
|
169
|
+
* write baseline.
|
|
170
|
+
*/
|
|
171
|
+
function diskLimitValues(text: string, warn: Warn = NO_WARN): Record<string, number> {
|
|
172
|
+
let parsed: unknown;
|
|
173
|
+
try {
|
|
174
|
+
parsed = YAML.parse(text);
|
|
175
|
+
} catch (err) {
|
|
176
|
+
warn(`[omp-opsx-addon] concurrency: config is not readable as YAML (${message(err)}); treating limits as absent`);
|
|
177
|
+
return {};
|
|
178
|
+
}
|
|
179
|
+
const limits = (parsed as { providers?: { maxInFlightRequests?: unknown } } | null | undefined)
|
|
180
|
+
?.providers?.maxInFlightRequests;
|
|
181
|
+
if (!limits || typeof limits !== 'object' || Array.isArray(limits)) return {};
|
|
182
|
+
const out: Record<string, number> = {};
|
|
183
|
+
for (const [id, value] of Object.entries(limits as Record<string, unknown>)) {
|
|
184
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) continue;
|
|
185
|
+
const canonical = canonicalizeProvider(id.trim());
|
|
186
|
+
if (!canonical) continue;
|
|
187
|
+
out[canonical] = value;
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Read the current shared value for each requested provider out of
|
|
194
|
+
* `configPath` (`providers.maxInFlightRequests`).
|
|
195
|
+
*
|
|
196
|
+
* Returns one row per requested provider, in request order, with the
|
|
197
|
+
* canonicalized provider id; `value: null` means「无现值」(key absent, or the
|
|
198
|
+
* entry is not a positive integer). A missing/unparseable config yields all
|
|
199
|
+
* `null`s — callers treat that as `initial`/`concurrency_start`, never as
|
|
200
|
+
* unlimited.
|
|
201
|
+
*/
|
|
202
|
+
export function readManagedLimits(
|
|
203
|
+
configPath: string,
|
|
204
|
+
providers: readonly string[],
|
|
205
|
+
fileSystem: WriterFileSystemPort = nodeWriterFileSystem,
|
|
206
|
+
warn: Warn = NO_WARN,
|
|
207
|
+
): ManagedLimitValue[] {
|
|
208
|
+
const text = fileSystem.readFile(configPath);
|
|
209
|
+
if (text === null) return providers.map((provider) => ({ provider: canonicalizeProvider(provider.trim()), value: null }));
|
|
210
|
+
const parsedValues = diskLimitValues(text, warn);
|
|
211
|
+
return providers.map((provider) => {
|
|
212
|
+
const canonical = canonicalizeProvider(provider.trim());
|
|
213
|
+
const value = parsedValues[canonical];
|
|
214
|
+
return { provider: canonical, value: typeof value === 'number' ? value : null };
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Merge one provider's new limit into the **complete** per-provider map.
|
|
220
|
+
* `providers.maxInFlightRequests` is a whole-key override on the host side, so
|
|
221
|
+
* every caller MUST pass the full map (this is the map that goes to
|
|
222
|
+
* `settings.override` and to {@link writeManagedLimit}).
|
|
223
|
+
*/
|
|
224
|
+
export function buildMergedMap(
|
|
225
|
+
current: Record<string, number>,
|
|
226
|
+
provider: string,
|
|
227
|
+
value: number,
|
|
228
|
+
): Record<string, number> {
|
|
229
|
+
const key = canonicalizeProvider(provider.trim()) || provider.trim();
|
|
230
|
+
return { ...current, [key]: value };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// ── write threshold (4.2) ───────────────────────────────────────────
|
|
234
|
+
|
|
235
|
+
export interface WriteGateArgs {
|
|
236
|
+
/** Target `L` for this evaluation window. */
|
|
237
|
+
newValue: number;
|
|
238
|
+
/** Shared current value; `null` when the provider has no key yet. */
|
|
239
|
+
oldValue: number | null;
|
|
240
|
+
/** Epoch ms of this provider's last successful write (`0`/`null` = never). */
|
|
241
|
+
lastWriteAt: number | null;
|
|
242
|
+
/** Current epoch ms (injected clock). */
|
|
243
|
+
now: number;
|
|
244
|
+
/** `concurrency_write_min_interval_ms`. */
|
|
245
|
+
minIntervalMs: number;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export interface WriteGateDecision {
|
|
249
|
+
write: boolean;
|
|
250
|
+
/** `'unchanged'` = `|Δ| < 1`; `'too-soon'` = within the write interval. */
|
|
251
|
+
reason?: 'unchanged' | 'too-soon';
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Write gate: a write needs `|new − old| >= 1` **and** more than
|
|
256
|
+
* `minIntervalMs` since this provider's last successful write. A provider with
|
|
257
|
+
* no current value (`oldValue === null`) has nothing to compare against, and a
|
|
258
|
+
* provider never written has no interval to respect — both pass the gate.
|
|
259
|
+
*/
|
|
260
|
+
export function shouldWriteLimit(args: WriteGateArgs): WriteGateDecision {
|
|
261
|
+
const { newValue, oldValue, lastWriteAt, now, minIntervalMs } = args;
|
|
262
|
+
if (oldValue !== null && Math.abs(newValue - oldValue) < 1) return { write: false, reason: 'unchanged' };
|
|
263
|
+
const interval = Number.isFinite(minIntervalMs) && minIntervalMs > 0 ? minIntervalMs : 0;
|
|
264
|
+
if (lastWriteAt !== null && lastWriteAt > 0 && now - lastWriteAt <= interval) return { write: false, reason: 'too-soon' };
|
|
265
|
+
return { write: true };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* `L` start value for a process that just acquired (or took over) the lease:
|
|
270
|
+
* the shared value clamped to `[floor, ceiling]`, falling back to
|
|
271
|
+
* `concurrency_start` only when no shared value exists. This is what keeps a
|
|
272
|
+
* handover from resurrecting the new writer's stale local trajectory.
|
|
273
|
+
*/
|
|
274
|
+
export function resolveTakeoverStart(
|
|
275
|
+
sharedValue: number | null,
|
|
276
|
+
options: { floor: number; ceiling: number; start: number },
|
|
277
|
+
): number {
|
|
278
|
+
const base = sharedValue === null ? options.start : sharedValue;
|
|
279
|
+
return clampLimit(base, options.floor, options.ceiling);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ── block splice: write only the target provider's value (4.3/4.8) ──
|
|
283
|
+
|
|
284
|
+
interface LimitEntry {
|
|
285
|
+
/** Entry line index. */
|
|
286
|
+
line: number;
|
|
287
|
+
/** Leading whitespace of the entry line. */
|
|
288
|
+
indent: string;
|
|
289
|
+
/** Raw key text as written (quotes preserved). */
|
|
290
|
+
rawId: string;
|
|
291
|
+
/** Canonical provider id. */
|
|
292
|
+
canonical: string;
|
|
293
|
+
/** Index where the value text starts / ends on the entry line. */
|
|
294
|
+
valueStart: number;
|
|
295
|
+
valueEnd: number;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
interface LimitBlock {
|
|
299
|
+
/** `providers:` line index / indent, or -1 when the key is absent. */
|
|
300
|
+
providersLine: number;
|
|
301
|
+
providersIndent: string;
|
|
302
|
+
/** Insertion point just past the last non-blank line of the `providers` block. */
|
|
303
|
+
providersEnd: number;
|
|
304
|
+
/** `maxInFlightRequests:` line index, or -1 when absent. */
|
|
305
|
+
keyLine: number;
|
|
306
|
+
keyIndent: string;
|
|
307
|
+
/** Text after the colon when the mapping is inline (`{}` / `{a: 1}`). */
|
|
308
|
+
inline: string;
|
|
309
|
+
entries: LimitEntry[];
|
|
310
|
+
/** Insertion point just past the last non-blank line of the mapping block. */
|
|
311
|
+
end: number;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const KEY_LINE_RE = /^(\s*)([^#\s][^:]*?)\s*:(.*)$/;
|
|
315
|
+
// Entry line: indent, key, separator, value token, untouched remainder (e.g. a
|
|
316
|
+
// trailing `# comment`) — only the value token is ever replaced.
|
|
317
|
+
const ENTRY_RE = /^(\s*)([^#\s][^:]*?)(\s*:\s*)(\S*)(.*)$/;
|
|
318
|
+
|
|
319
|
+
function indentWidth(line: string): number {
|
|
320
|
+
const m = /^\s*/.exec(line);
|
|
321
|
+
return m ? m[0].length : 0;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Locate `providers.maxInFlightRequests` and its entries in the raw text. */
|
|
325
|
+
function parseLimitBlock(text: string): LimitBlock {
|
|
326
|
+
const lines = text.split('\n');
|
|
327
|
+
const block: LimitBlock = {
|
|
328
|
+
providersLine: -1,
|
|
329
|
+
providersIndent: '',
|
|
330
|
+
providersEnd: 0,
|
|
331
|
+
keyLine: -1,
|
|
332
|
+
keyIndent: '',
|
|
333
|
+
inline: '',
|
|
334
|
+
entries: [],
|
|
335
|
+
end: 0,
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
for (let i = 0; i < lines.length; i++) {
|
|
339
|
+
const m = KEY_LINE_RE.exec(lines[i]);
|
|
340
|
+
if (!m || m[1].length !== 0 || m[2].trim() !== 'providers') continue;
|
|
341
|
+
block.providersLine = i;
|
|
342
|
+
block.providersIndent = m[1];
|
|
343
|
+
// Last non-blank line of the `providers` block (its children have a deeper indent).
|
|
344
|
+
block.providersEnd = i + 1;
|
|
345
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
346
|
+
if (indentWidth(lines[j]) <= 0 && lines[j].trim() !== '') break;
|
|
347
|
+
if (lines[j].trim() !== '') block.providersEnd = j + 1;
|
|
348
|
+
}
|
|
349
|
+
break;
|
|
350
|
+
}
|
|
351
|
+
if (block.providersLine < 0) return block;
|
|
352
|
+
|
|
353
|
+
for (let i = block.providersLine + 1; i < lines.length; i++) {
|
|
354
|
+
const line = lines[i];
|
|
355
|
+
if (line.trim() !== '' && indentWidth(line) <= block.providersIndent.length) break;
|
|
356
|
+
const m = KEY_LINE_RE.exec(line);
|
|
357
|
+
if (!m || m[2].trim() !== 'maxInFlightRequests') continue;
|
|
358
|
+
block.keyLine = i;
|
|
359
|
+
block.keyIndent = m[1];
|
|
360
|
+
block.inline = m[3].trim();
|
|
361
|
+
block.end = i + 1;
|
|
362
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
363
|
+
const entryLine = lines[j];
|
|
364
|
+
if (entryLine.trim() !== '' && indentWidth(entryLine) <= m[1].length) break;
|
|
365
|
+
if (entryLine.trim() === '') continue;
|
|
366
|
+
block.end = j + 1;
|
|
367
|
+
const em = ENTRY_RE.exec(entryLine);
|
|
368
|
+
if (!em) continue; // comment or nested mapping — preserved verbatim, not an entry
|
|
369
|
+
const rawId = em[2].trim();
|
|
370
|
+
const valueStart = em[1].length + em[2].length + em[3].length;
|
|
371
|
+
const valueEnd = valueStart + em[4].length;
|
|
372
|
+
block.entries.push({
|
|
373
|
+
line: j,
|
|
374
|
+
indent: em[1],
|
|
375
|
+
rawId,
|
|
376
|
+
canonical: canonicalizeProvider(rawId.replace(/^["']|["']$/g, '').trim()),
|
|
377
|
+
valueStart,
|
|
378
|
+
valueEnd,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
break;
|
|
382
|
+
}
|
|
383
|
+
return block;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Splice `values` into the limits mapping of `text`, returning the new text.
|
|
388
|
+
* Existing entry lines are rewritten **only** when their value actually
|
|
389
|
+
* changes (so untouched providers stay byte-identical); new providers are
|
|
390
|
+
* appended at the end of the mapping; a block/inline/absent mapping is
|
|
391
|
+
* normalized to block style. Nothing outside the mapping is touched.
|
|
392
|
+
*/
|
|
393
|
+
function renderLimitBlock(text: string, block: LimitBlock, values: Map<string, number>): string {
|
|
394
|
+
const lines = text.split('\n');
|
|
395
|
+
const childIndent = `${block.keyIndent} `;
|
|
396
|
+
|
|
397
|
+
// Mapping key absent entirely → create it (and `providers:` when needed).
|
|
398
|
+
if (block.keyLine < 0) {
|
|
399
|
+
const entries = [...values].map(([id, value]) => `${childIndent} ${id}: ${value}`);
|
|
400
|
+
const created = [`${childIndent}maxInFlightRequests:`, ...entries];
|
|
401
|
+
if (block.providersLine >= 0) {
|
|
402
|
+
lines.splice(block.providersEnd, 0, ...created);
|
|
403
|
+
return lines.join('\n');
|
|
404
|
+
}
|
|
405
|
+
const at = lines.length > 0 && lines[lines.length - 1] === '' ? lines.length - 1 : lines.length;
|
|
406
|
+
const sep = lines.length > 0 && lines[0] !== '' ? [''] : [];
|
|
407
|
+
lines.splice(at, 0, ...sep, 'providers:', ...created);
|
|
408
|
+
return lines.join('\n');
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// Inline mapping (`maxInFlightRequests: {}` / `{a: 1}`): expand to block
|
|
412
|
+
// style so entries can be updated textually. Values are preserved; only
|
|
413
|
+
// this one line's formatting changes.
|
|
414
|
+
if (block.inline !== '') {
|
|
415
|
+
const entries = [...values].map(([id, value]) => `${childIndent}${id}: ${value}`);
|
|
416
|
+
lines.splice(block.keyLine, 1, `${block.keyIndent}maxInFlightRequests:`, ...entries);
|
|
417
|
+
return lines.join('\n');
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const present = new Set(block.entries.map((entry) => entry.canonical));
|
|
421
|
+
for (const entry of block.entries) {
|
|
422
|
+
const value = values.get(entry.canonical);
|
|
423
|
+
if (value === undefined) continue;
|
|
424
|
+
const line = lines[entry.line];
|
|
425
|
+
const rebuilt = `${line.slice(0, entry.valueStart)}${value}${line.slice(entry.valueEnd)}`;
|
|
426
|
+
if (rebuilt !== line) lines[entry.line] = rebuilt;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
const extra = [...values].filter(([id]) => !present.has(id));
|
|
430
|
+
if (extra.length > 0) {
|
|
431
|
+
const entryIndent = block.entries.length > 0 ? block.entries[0].indent : childIndent;
|
|
432
|
+
const appended = extra.map(([id, value]) => `${entryIndent}${id}: ${value}`);
|
|
433
|
+
lines.splice(block.end, 0, ...appended);
|
|
434
|
+
}
|
|
435
|
+
return lines.join('\n');
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// ── cross-process write lock, mirroring lib/usage-poller.ts (4.3) ───
|
|
439
|
+
|
|
440
|
+
type LockOutcome = { release: () => void } | { reason: 'lock-busy' | 'lock-unavailable' };
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* Exclusive `<file>.lock` acquire: `O_EXCL` create, steal locks older than
|
|
444
|
+
* {@link WRITE_LOCK_STALE_MS}, retry briefly while a live holder exists (the
|
|
445
|
+
* lock queues plugin writers; the host's own writes converge via read-merge-write
|
|
446
|
+
* + atomic rename instead of this lock). Returns a release fn, or the reason
|
|
447
|
+
* the caller must skip the write.
|
|
448
|
+
*/
|
|
449
|
+
function acquireWriteLock(
|
|
450
|
+
lockPath: string,
|
|
451
|
+
fileSystem: WriterFileSystemPort,
|
|
452
|
+
now: number,
|
|
453
|
+
sleep: (ms: number) => void,
|
|
454
|
+
): LockOutcome {
|
|
455
|
+
for (let attempt = 0; attempt < WRITE_LOCK_ATTEMPTS; attempt++) {
|
|
456
|
+
try {
|
|
457
|
+
fileSystem.openExclusive(lockPath, String(process.pid));
|
|
458
|
+
return {
|
|
459
|
+
release: () => {
|
|
460
|
+
try { fileSystem.remove(lockPath); } catch { /* already gone */ }
|
|
461
|
+
},
|
|
462
|
+
};
|
|
463
|
+
} catch (err) {
|
|
464
|
+
if (codeOf(err) !== 'EEXIST') {
|
|
465
|
+
// Not contention: this file system cannot do atomic creation.
|
|
466
|
+
return { reason: 'lock-unavailable' };
|
|
467
|
+
}
|
|
468
|
+
const mtime = fileSystem.statMtimeMs(lockPath);
|
|
469
|
+
if (mtime === null) continue; // lock vanished between create and stat — retry now
|
|
470
|
+
if (now - mtime > WRITE_LOCK_STALE_MS) {
|
|
471
|
+
try { fileSystem.remove(lockPath); } catch { /* raced */ }
|
|
472
|
+
continue;
|
|
473
|
+
}
|
|
474
|
+
if (attempt < WRITE_LOCK_ATTEMPTS - 1) sleep(WRITE_LOCK_RETRY_MS);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
return { reason: 'lock-busy' };
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// ── read-merge-write (4.3) + settings hot apply (4.4) ───────────────
|
|
481
|
+
|
|
482
|
+
export interface WriteManagedLimitArgs {
|
|
483
|
+
/** Config root `config.yml`. */
|
|
484
|
+
configPath: string;
|
|
485
|
+
/** The one provider this write changes. */
|
|
486
|
+
provider: string;
|
|
487
|
+
/**
|
|
488
|
+
* Intended map (see {@link buildMergedMap}). **Only `merged[provider]` is
|
|
489
|
+
* used**: every other key comes from the in-lock re-read, so a stale caller
|
|
490
|
+
* snapshot cannot roll back a peer's change to another provider.
|
|
491
|
+
*/
|
|
492
|
+
merged: Record<string, number>;
|
|
493
|
+
/** Write-lock path; defaults to `<configPath>.lock`. */
|
|
494
|
+
lockPath?: string;
|
|
495
|
+
/** Epoch ms used for lock staleness (tests inject it); defaults to `Date.now()`. */
|
|
496
|
+
now?: number;
|
|
497
|
+
fileSystem?: WriterFileSystemPort;
|
|
498
|
+
/** Sleep used between lock retries; defaults to a synchronous sleep. */
|
|
499
|
+
sleep?: (ms: number) => void;
|
|
500
|
+
warn?: Warn;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
export interface WriteManagedLimitResult {
|
|
504
|
+
written: boolean;
|
|
505
|
+
/** Why nothing was written: `unchanged` | `config-missing` | `invalid-provider` | `invalid-value` | `lock-busy` | `lock-unavailable` | `write-failed`. */
|
|
506
|
+
reason?: string;
|
|
507
|
+
/** The merged map that was persisted (canonical provider → value). */
|
|
508
|
+
limits?: Record<string, number>;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
let writeSeq = 0;
|
|
512
|
+
|
|
513
|
+
/**
|
|
514
|
+
* Locked atomic read-merge-write of `providers.maxInFlightRequests`.
|
|
515
|
+
*
|
|
516
|
+
* Inside the lock the file is re-read and only the **target** provider's value
|
|
517
|
+
* is taken from the caller; every other key keeps its freshly-read on-disk
|
|
518
|
+
* value. The caller's map is a snapshot from earlier in the evaluation tick, so
|
|
519
|
+
* applying it wholesale would roll back a peer's concurrent change to a
|
|
520
|
+
* *different* provider — the whole point of read-merge-write. Only the changed
|
|
521
|
+
* entry lines are rewritten, so the rest of the file stays byte-identical and a
|
|
522
|
+
* host write to another key between our read and our write is never lost. The
|
|
523
|
+
* payload lands via `tmp` + `rename`. When the merge yields no textual change,
|
|
524
|
+
* nothing is written at all (no tmp file, no mtime bump).
|
|
525
|
+
*/
|
|
526
|
+
export function writeManagedLimit(args: WriteManagedLimitArgs): WriteManagedLimitResult {
|
|
527
|
+
const fileSystem = args.fileSystem ?? nodeWriterFileSystem;
|
|
528
|
+
const warn = args.warn ?? NO_WARN;
|
|
529
|
+
const configPath = args.configPath;
|
|
530
|
+
const now = args.now ?? Date.now();
|
|
531
|
+
|
|
532
|
+
if (fileSystem.readFile(configPath) === null) {
|
|
533
|
+
warn(`[omp-opsx-addon] concurrency: config ${configPath} is not readable; limit write skipped`);
|
|
534
|
+
return { written: false, reason: 'config-missing' };
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
const lock = acquireWriteLock(args.lockPath ?? `${configPath}.lock`, fileSystem, now, args.sleep ?? sleepSync);
|
|
538
|
+
if ('reason' in lock) {
|
|
539
|
+
warn(
|
|
540
|
+
lock.reason === 'lock-busy'
|
|
541
|
+
? `[omp-opsx-addon] concurrency: write lock ${args.lockPath ?? `${configPath}.lock`} is held; limit write skipped (retried next window)`
|
|
542
|
+
: `[omp-opsx-addon] concurrency: atomic lock creation unavailable; limit write skipped`,
|
|
543
|
+
);
|
|
544
|
+
return { written: false, reason: lock.reason };
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
try {
|
|
548
|
+
const current = fileSystem.readFile(configPath);
|
|
549
|
+
if (current === null) {
|
|
550
|
+
warn(`[omp-opsx-addon] concurrency: config ${configPath} became unreadable; limit write skipped`);
|
|
551
|
+
return { written: false, reason: 'config-missing' };
|
|
552
|
+
}
|
|
553
|
+
const target = canonicalizeProvider(String(args.provider ?? '').trim());
|
|
554
|
+
if (!target) {
|
|
555
|
+
warn('[omp-opsx-addon] concurrency: limit write skipped — no target provider given');
|
|
556
|
+
return { written: false, reason: 'invalid-provider' };
|
|
557
|
+
}
|
|
558
|
+
const targetValue = (args.merged ?? {})[target];
|
|
559
|
+
if (typeof targetValue !== 'number' || !Number.isSafeInteger(targetValue) || targetValue < 1) {
|
|
560
|
+
warn(`[omp-opsx-addon] concurrency: skipping ${target}: merged value ${JSON.stringify(targetValue)} is not a positive integer`);
|
|
561
|
+
return { written: false, reason: 'invalid-value' };
|
|
562
|
+
}
|
|
563
|
+
const values = new Map<string, number>(Object.entries(diskLimitValues(current)));
|
|
564
|
+
values.set(target, targetValue);
|
|
565
|
+
|
|
566
|
+
const block = parseLimitBlock(current);
|
|
567
|
+
const next = renderLimitBlock(current, block, values);
|
|
568
|
+
const limits = Object.fromEntries(values);
|
|
569
|
+
if (next === current) return { written: false, reason: 'unchanged', limits };
|
|
570
|
+
|
|
571
|
+
fileSystem.ensureDir(dirname(configPath));
|
|
572
|
+
const tmp = `${configPath}.tmp-${process.pid}-${writeSeq++}`;
|
|
573
|
+
fileSystem.writeFile(tmp, next);
|
|
574
|
+
fileSystem.rename(tmp, configPath);
|
|
575
|
+
return { written: true, limits };
|
|
576
|
+
} catch (err) {
|
|
577
|
+
warn(`[omp-opsx-addon] concurrency: limit write to ${configPath} failed (${message(err)}); keeping the previous value`);
|
|
578
|
+
return { written: false, reason: 'write-failed' };
|
|
579
|
+
} finally {
|
|
580
|
+
lock.release();
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Injected host settings port (`pi.pi.settings`), narrowed to the call we make.
|
|
586
|
+
*/
|
|
587
|
+
export interface SettingsPort {
|
|
588
|
+
override(path: string, value: unknown): void;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
export interface ApplyLimitArgs {
|
|
592
|
+
settings: SettingsPort;
|
|
593
|
+
/** Complete per-provider map (see {@link buildMergedMap}). */
|
|
594
|
+
merged: Record<string, number>;
|
|
595
|
+
provider: string;
|
|
596
|
+
value: number;
|
|
597
|
+
warn?: Warn;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
export interface ApplyLimitResult {
|
|
601
|
+
applied: boolean;
|
|
602
|
+
/** `invalid-provider` | `invalid-merged` | `settings-unavailable` | `override-failed`. */
|
|
603
|
+
reason?: string;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Hot-apply the shared map in this process via
|
|
608
|
+
* `settings.override('providers.maxInFlightRequests', merged)`.
|
|
609
|
+
*
|
|
610
|
+
* `providers.maxInFlightRequests` is a **whole-key override**, so the second
|
|
611
|
+
* argument MUST be the complete per-provider map — passing only the target
|
|
612
|
+
* provider's value would clear every other provider's limit. The map is
|
|
613
|
+
* validated before the call: refusing an all-or-nothing override is safe
|
|
614
|
+
* (the caller keeps the previous value), while a partial map would silently
|
|
615
|
+
* make the omitted providers unlimited. When the API throws (host schema moved
|
|
616
|
+
* on), the persisted config is untouched — the host config watcher picks the
|
|
617
|
+
* value up — and the failure is warned, never fatal.
|
|
618
|
+
*/
|
|
619
|
+
export function applyLimit(args: ApplyLimitArgs): ApplyLimitResult {
|
|
620
|
+
const warn = args.warn ?? NO_WARN;
|
|
621
|
+
const settings = args.settings;
|
|
622
|
+
if (!settings || typeof settings.override !== 'function') {
|
|
623
|
+
warn('[omp-opsx-addon] concurrency: settings.override unavailable; skipping hot apply (config file value still applies)');
|
|
624
|
+
return { applied: false, reason: 'settings-unavailable' };
|
|
625
|
+
}
|
|
626
|
+
const provider = canonicalizeProvider(args.provider.trim());
|
|
627
|
+
if (!provider) {
|
|
628
|
+
warn('[omp-opsx-addon] concurrency: cannot hot-apply an empty provider id');
|
|
629
|
+
return { applied: false, reason: 'invalid-provider' };
|
|
630
|
+
}
|
|
631
|
+
const payload = buildMergedMap(args.merged ?? {}, provider, args.value);
|
|
632
|
+
for (const [id, value] of Object.entries(payload)) {
|
|
633
|
+
if (!id || typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) {
|
|
634
|
+
warn(`[omp-opsx-addon] concurrency: refusing settings.override — invalid map entry ${JSON.stringify(id)}: ${JSON.stringify(value)}`);
|
|
635
|
+
return { applied: false, reason: 'invalid-merged' };
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
try {
|
|
639
|
+
settings.override('providers.maxInFlightRequests', payload);
|
|
640
|
+
return { applied: true };
|
|
641
|
+
} catch (err) {
|
|
642
|
+
warn(`[omp-opsx-addon] concurrency: settings.override failed (${message(err)}); the host config watcher will apply the persisted value`);
|
|
643
|
+
return { applied: false, reason: 'override-failed' };
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// ── writer lease: Redis NX/PX → file lease → read-only (4.5–4.7) ────
|
|
648
|
+
|
|
649
|
+
/** Redis lease mechanism: satisfied by `lib/usage-redis-client.ts`'s `RedisClient`. */
|
|
650
|
+
export interface LeaseRedisPort {
|
|
651
|
+
pipeline(commands: string[][]): Promise<readonly unknown[]>;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
export type LeaseMode = 'redis' | 'file' | 'readonly';
|
|
655
|
+
|
|
656
|
+
export interface CreateLeaseArgs {
|
|
657
|
+
/** Redis client, or `null`/absent to go straight to the file lease. */
|
|
658
|
+
redis?: LeaseRedisPort | null;
|
|
659
|
+
/** File-lease path (config root), reused for takeover staleness. */
|
|
660
|
+
lockPath: string;
|
|
661
|
+
/** Lease owner: `pid` + process start time. */
|
|
662
|
+
owner: string;
|
|
663
|
+
/** `concurrency_lease_ttl_ms`. */
|
|
664
|
+
ttlMs: number;
|
|
665
|
+
/** Injected clock (epoch ms). */
|
|
666
|
+
now: () => number;
|
|
667
|
+
/** Redis key override; defaults to {@link defaultLeaseKey}. */
|
|
668
|
+
redisKey?: string;
|
|
669
|
+
fileSystem?: WriterFileSystemPort;
|
|
670
|
+
warn?: Warn;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Lease handle. `held`/`mode` are a snapshot of the acquisition; `renew()`
|
|
675
|
+
* returns `false` once the lease was lost (takeover by a peer, TTL expiry, or a
|
|
676
|
+
* Redis failure) and the holder MUST stop writing.
|
|
677
|
+
*/
|
|
678
|
+
export interface LeaseHandle {
|
|
679
|
+
held: boolean;
|
|
680
|
+
mode: LeaseMode;
|
|
681
|
+
/**
|
|
682
|
+
* Owner id of the lease as this process last observed it: its own `owner`
|
|
683
|
+
* while held, the peer's id when a peer holds it, `null` when unknown
|
|
684
|
+
* (nobody holds it, or the mechanism cannot report it). Read-only diagnostic
|
|
685
|
+
* surface — a non-writer's summary shows *who* holds the lease, not just
|
|
686
|
+
* that it is not this process.
|
|
687
|
+
*/
|
|
688
|
+
readonly holderId: string | null;
|
|
689
|
+
renew(): Promise<boolean>;
|
|
690
|
+
release(): Promise<void>;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const NOT_HELD = (mode: LeaseMode, holderId: string | null = null): LeaseHandle => ({
|
|
694
|
+
held: false,
|
|
695
|
+
mode,
|
|
696
|
+
holderId,
|
|
697
|
+
renew: async () => false,
|
|
698
|
+
release: async () => {},
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Redis lease key for one config root, derived from the lease file's directory
|
|
703
|
+
* (`opsx:concurrency:tuner:<configRootHash>`).
|
|
704
|
+
*/
|
|
705
|
+
export function defaultLeaseKey(lockPath: string): string {
|
|
706
|
+
const root = dirname(lockPath);
|
|
707
|
+
// FNV-1a 32-bit — stable across processes, no crypto import needed.
|
|
708
|
+
let hash = 0x811c9dc5;
|
|
709
|
+
for (let i = 0; i < root.length; i++) {
|
|
710
|
+
hash ^= root.charCodeAt(i);
|
|
711
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
712
|
+
}
|
|
713
|
+
return `opsx:concurrency:tuner:${hash.toString(16).padStart(8, '0')}`;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
function redisLease(
|
|
717
|
+
redis: LeaseRedisPort,
|
|
718
|
+
key: string,
|
|
719
|
+
owner: string,
|
|
720
|
+
ttlMs: number,
|
|
721
|
+
now: () => number,
|
|
722
|
+
warn: Warn,
|
|
723
|
+
): LeaseHandle {
|
|
724
|
+
let held = true;
|
|
725
|
+
let holderId: string | null = owner;
|
|
726
|
+
let expiresAt = now() + ttlMs;
|
|
727
|
+
return {
|
|
728
|
+
held: true,
|
|
729
|
+
mode: 'redis',
|
|
730
|
+
get holderId(): string | null {
|
|
731
|
+
// While held, the holder is this process; after stepping down the
|
|
732
|
+
// field carries the last owner we observed (the peer, or `null`).
|
|
733
|
+
return held ? owner : holderId;
|
|
734
|
+
},
|
|
735
|
+
async renew(): Promise<boolean> {
|
|
736
|
+
if (!held) return false;
|
|
737
|
+
// Past our own estimate the key may already have been taken over:
|
|
738
|
+
// renewing then would steal the lease back from a live peer.
|
|
739
|
+
if (now() >= expiresAt) {
|
|
740
|
+
held = false;
|
|
741
|
+
holderId = null;
|
|
742
|
+
return false;
|
|
743
|
+
}
|
|
744
|
+
try {
|
|
745
|
+
// Compare-then-extend. `SET ... XX` alone would also refresh a
|
|
746
|
+
// peer's lease if ours expired unnoticed in between.
|
|
747
|
+
const current = (await redis.pipeline([['GET', key]]))[0];
|
|
748
|
+
if (current !== owner) {
|
|
749
|
+
// Remember *who* holds it now so the summary can name the peer.
|
|
750
|
+
holderId = typeof current === 'string' && current !== '' ? current : null;
|
|
751
|
+
held = false;
|
|
752
|
+
return false;
|
|
753
|
+
}
|
|
754
|
+
const reply = (await redis.pipeline([['SET', key, owner, 'PX', String(ttlMs), 'XX']]))[0];
|
|
755
|
+
if (reply !== 'OK' && reply !== 'ok') {
|
|
756
|
+
held = false;
|
|
757
|
+
holderId = null;
|
|
758
|
+
return false;
|
|
759
|
+
}
|
|
760
|
+
expiresAt = now() + ttlMs;
|
|
761
|
+
return true;
|
|
762
|
+
} catch (err) {
|
|
763
|
+
warn(`[omp-opsx-addon] concurrency: lease renewal failed (${message(err)}); stepping down to non-writer`);
|
|
764
|
+
held = false;
|
|
765
|
+
holderId = null;
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
},
|
|
769
|
+
async release(): Promise<void> {
|
|
770
|
+
if (!held) return;
|
|
771
|
+
held = false;
|
|
772
|
+
holderId = null;
|
|
773
|
+
try {
|
|
774
|
+
await redis.pipeline([['DEL', key]]);
|
|
775
|
+
} catch { /* TTL will clean it up */ }
|
|
776
|
+
},
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
function fileLeaseHandle(
|
|
781
|
+
fileSystem: WriterFileSystemPort,
|
|
782
|
+
lockPath: string,
|
|
783
|
+
owner: string,
|
|
784
|
+
ttlMs: number,
|
|
785
|
+
now: () => number,
|
|
786
|
+
warn: Warn,
|
|
787
|
+
): LeaseHandle {
|
|
788
|
+
let held = false;
|
|
789
|
+
let holderId: string | null = null;
|
|
790
|
+
let unsupported: string | null = null;
|
|
791
|
+
for (let attempt = 0; attempt < FILE_LEASE_ATTEMPTS; attempt++) {
|
|
792
|
+
try {
|
|
793
|
+
fileSystem.openExclusive(lockPath, owner);
|
|
794
|
+
held = true;
|
|
795
|
+
break;
|
|
796
|
+
} catch (err) {
|
|
797
|
+
if (codeOf(err) !== 'EEXIST') {
|
|
798
|
+
unsupported = codeOf(err) ?? message(err);
|
|
799
|
+
break;
|
|
800
|
+
}
|
|
801
|
+
const mtime = fileSystem.statMtimeMs(lockPath);
|
|
802
|
+
if (mtime === null) continue; // vanished — retry immediately
|
|
803
|
+
if (now() - mtime > ttlMs) {
|
|
804
|
+
try { fileSystem.remove(lockPath); } catch { /* raced */ }
|
|
805
|
+
continue; // stale holder → take over
|
|
806
|
+
}
|
|
807
|
+
break; // a live peer holds the lease
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
if (!held) {
|
|
811
|
+
if (unsupported !== null) {
|
|
812
|
+
warn(`[omp-opsx-addon] concurrency: file lease unavailable (${unsupported}) and Redis unusable; control loop degraded to read-only`);
|
|
813
|
+
return NOT_HELD('readonly');
|
|
814
|
+
}
|
|
815
|
+
// A live peer holds the lock: its owner id is the lock file's body, so a
|
|
816
|
+
// non-writer can name the holder instead of reporting an anonymous「有主」.
|
|
817
|
+
const peer = fileSystem.readFile(lockPath);
|
|
818
|
+
return NOT_HELD('file', typeof peer === 'string' && peer !== '' ? peer : null);
|
|
819
|
+
}
|
|
820
|
+
return {
|
|
821
|
+
held: true,
|
|
822
|
+
mode: 'file',
|
|
823
|
+
get holderId(): string | null {
|
|
824
|
+
return held ? owner : holderId;
|
|
825
|
+
},
|
|
826
|
+
async renew(): Promise<boolean> {
|
|
827
|
+
if (!held) return false;
|
|
828
|
+
const current = fileSystem.readFile(lockPath);
|
|
829
|
+
if (current !== owner) {
|
|
830
|
+
held = false;
|
|
831
|
+
holderId = typeof current === 'string' && current !== '' ? current : null;
|
|
832
|
+
return false;
|
|
833
|
+
}
|
|
834
|
+
try {
|
|
835
|
+
fileSystem.touch(lockPath, now());
|
|
836
|
+
return true;
|
|
837
|
+
} catch (err) {
|
|
838
|
+
warn(`[omp-opsx-addon] concurrency: file lease renewal failed (${message(err)}); stepping down to non-writer`);
|
|
839
|
+
held = false;
|
|
840
|
+
holderId = null;
|
|
841
|
+
return false;
|
|
842
|
+
}
|
|
843
|
+
},
|
|
844
|
+
async release(): Promise<void> {
|
|
845
|
+
if (!held) return;
|
|
846
|
+
held = false;
|
|
847
|
+
holderId = null;
|
|
848
|
+
if (fileSystem.readFile(lockPath) === owner) {
|
|
849
|
+
try { fileSystem.remove(lockPath); } catch { /* already gone */ }
|
|
850
|
+
}
|
|
851
|
+
},
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* Acquire writer eligibility for this process.
|
|
857
|
+
*
|
|
858
|
+
* 1. Redis available → `SET <key> <owner> NX PX <ttl>` through the existing
|
|
859
|
+
* `pipeline()` command surface (no client-command extension); a `null`
|
|
860
|
+
* reply means a peer holds the lease → `held: false`, `mode: 'redis'`.
|
|
861
|
+
* 2. Redis absent/failing → file lease at `lockPath` (`O_EXCL` + mtime expiry
|
|
862
|
+
* takeover) → `mode: 'file'`.
|
|
863
|
+
* 3. Neither available → `held: false`, `mode: 'readonly'` **plus warn**: the
|
|
864
|
+
* caller MUST NOT write anything and MUST NOT drop managed keys.
|
|
865
|
+
*/
|
|
866
|
+
export async function createLease(args: CreateLeaseArgs): Promise<LeaseHandle> {
|
|
867
|
+
const fileSystem = args.fileSystem ?? nodeWriterFileSystem;
|
|
868
|
+
const warn = args.warn ?? NO_WARN;
|
|
869
|
+
const ttlMs = Number.isFinite(args.ttlMs) && args.ttlMs > 0 ? args.ttlMs : DEFAULT_LEASE_TTL_MS;
|
|
870
|
+
const owner = args.owner;
|
|
871
|
+
|
|
872
|
+
if (args.redis && typeof args.redis.pipeline === 'function') {
|
|
873
|
+
const key = args.redisKey ?? defaultLeaseKey(args.lockPath);
|
|
874
|
+
try {
|
|
875
|
+
const reply = (await args.redis.pipeline([['SET', key, owner, 'NX', 'PX', String(ttlMs)]]))[0];
|
|
876
|
+
if (reply === 'OK' || reply === 'ok') return redisLease(args.redis, key, owner, ttlMs, args.now, warn);
|
|
877
|
+
// A peer won the race: `GET` names it so the summary shows *who*
|
|
878
|
+
// holds the lease rather than an anonymous non-writer.
|
|
879
|
+
let holder: string | null = null;
|
|
880
|
+
try {
|
|
881
|
+
const current = (await args.redis.pipeline([['GET', key]]))[0];
|
|
882
|
+
holder = typeof current === 'string' && current !== '' ? current : null;
|
|
883
|
+
} catch { /* holder unknown — the lease still is not ours */ }
|
|
884
|
+
return NOT_HELD('redis', holder);
|
|
885
|
+
} catch (err) {
|
|
886
|
+
warn(`[omp-opsx-addon] concurrency: Redis lease unavailable (${message(err)}); falling back to the file lease`);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
try {
|
|
891
|
+
fileSystem.ensureDir(dirname(args.lockPath));
|
|
892
|
+
} catch { /* the open below decides whether atomic creation works */ }
|
|
893
|
+
return fileLeaseHandle(fileSystem, args.lockPath, owner, ttlMs, args.now, warn);
|
|
894
|
+
}
|