@genee/omp-opsx-addon 0.7.0 → 0.9.0

package/lib/concurrency-multiproc.worker.ts

@@ -0,0 +1,376 @@
+/**
+ * Worker process for `lib/concurrency-multiproc.test.ts` (tasks 7.2, spec
+ * Requirement「并发上限写入与跨进程单写者」).
+ *
+ * One worker == one OMP-like process running the *real* control loop
+ * (`ConcurrencyController` + the writer's `createLease` / read-merge-write)
+ * against a shared temp config root and a shared mock-Redis TCP server owned by
+ * the parent test.
+ *
+ * Synchronisation model — the parent fully drives the timeline:
+ *
+ *  - stdin: one JSON command per line (`{ id, cmd, ... }`).
+ *  - stdout: exactly one JSON reply per command, in order, tagged with the
+ *    command's `id`.
+ *  - the parent awaits each reply before sending the next command, so every
+ *    ordering fact (who holds the lease, that a write has landed, that a 429
+ *    was reported) is established by a request/response handshake instead of a
+ *    sleep. Nothing in this file polls or waits on wall-clock time.
+ *
+ * Clock: the control loop's clock is *injected* (`now: () => clock`) and only
+ * moves when the parent asks for it (`tick.advanceMs`), so `onTick`'s
+ * absolute-time gate, the AIMD windows and the write gate all advance
+ * deterministically. The base (`init.startAt`) is the parent's real `Date.now()`
+ * so that the writer's `<configPath>.lock` staleness comparison (real file
+ * mtime vs. injected `now`) stays meaningful.
+ *
+ * Commands:
+ *  - `init`     – build the controller (ports: node fs, no model_perf, optional
+ *                 Redis TCP client, recording settings port). With
+ *                 `brokenFileLease` the file-lease `O_EXCL` create always fails
+ *                 with `EOPNOTSUPP` (the "atomic create unavailable" injection).
+ *  - `tick`     – advance the clock by `advanceMs` and run one `onTick()`,
+ *                 replying with the tick result, the writer identity, every
+ *                 provider's control state, the on-disk shared values, the
+ *                 settings payloads and the warnings produced since last reply.
+ *  - `failure`  – deliver one in-process explicit failure (① classifier) via
+ *                 `noteExplicitFailure`, exactly like the host's 429 path would.
+ *  - `perf`     – advance the synthetic `model_perf` counters by one traffic
+ *                 step, i.e. make the *next* evaluation window carry a
+ *                 generation sample (R1 observation evidence; without it a
+ *                 window is frozen and `L` never moves up).
+ */
+
+import { TUNER_DEFAULTS } from './concurrency-tuner.js';
+import type { BadEventKind, ModelPerfRow, TunerOptions } from './concurrency-tuner.js';
+import {
+    createConcurrencyController,
+    createNodeFsPort,
+    type ConcurrencyController,
+    type ConcurrencyLeaseArgs,
+    type ConcurrencyLeaseFactory,
+} from './concurrency-control.js';
+import {
+    createLease,
+    nodeWriterFileSystem,
+    readManagedLimits,
+    type LeaseMode,
+    type SettingsPort,
+    type WriterFileSystemPort,
+} from './concurrency-writer.js';
+import { RedisClient } from './usage-redis-client.js';
+
+// ── wire protocol ───────────────────────────────────────────────────────────
+
+export interface MultiprocInitCommand {
+    id: number;
+    cmd: 'init';
+    /** Config root `config.yml`. */
+    configPath: string;
+    /** Managed providers (writer targets). */
+    providers: string[];
+    /** Lease owner id (`pid-<name>` from the parent). */
+    owner: string;
+    /** Injected clock base — the parent's real `Date.now()`. */
+    startAt: number;
+    /** Mock-Redis port, or `null` to run without the aggregation channel. */
+    redisPort: number | null;
+    /** Log directory the collector tails (empty in these tests). */
+    logDir: string;
+    tuner?: Partial<TunerOptions>;
+    leaseTtlMs?: number;
+    writeMinIntervalMs?: number;
+    /** `concurrency_ceiling_by_provider` — pins a provider at its fixture value. */
+    ceilingByProvider?: Record<string, number>;
+    /** Make the file-lease `O_EXCL` create fail (`EOPNOTSUPP`) → read-only path. */
+    brokenFileLease?: boolean;
+}
+
+export interface MultiprocTickCommand {
+    id: number;
+    cmd: 'tick';
+    /** Milliseconds to advance the injected clock before evaluating. */
+    advanceMs: number;
+}
+
+export interface MultiprocFailureCommand {
+    id: number;
+    cmd: 'failure';
+    provider: string;
+    /** Raw error/response text fed to the ① classifier (`429 …` = rate-limit). */
+    text: string;
+}
+
+export interface MultiprocPerfCommand {
+    id: number;
+    cmd: 'perf';
+    provider: string;
+    /** Δ samples for this step (default 10). */
+    samples?: number;
+    /** Δ output tokens (default 1_000). */
+    outputTokens?: number;
+    /** Δ generation time (default 1_000ms → 100ms per sample, a stable baseline). */
+    genMs?: number;
+}
+
+export type MultiprocCommand =
+    | MultiprocInitCommand
+    | MultiprocTickCommand
+    | MultiprocFailureCommand
+    | MultiprocPerfCommand;
+
+/** One managed provider's state as reported after a tick. */
+export interface MultiprocProviderView {
+    limit: number;
+    initial: number;
+    floor: number;
+    ceiling: number;
+    cleanWindows: number;
+    badStreak: number;
+    cooldownUntil: number;
+    badKind: BadEventKind | null;
+    increased: boolean;
+    decreased: boolean;
+    queueSuppressed: boolean;
+    cooldownActive: boolean;
+    rateLimitCount: number;
+    timeoutCount: number;
+    /** `true` only when this process was the writer *and* the limit landed on disk. */
+    written: boolean;
+}
+
+export interface MultiprocTickPayload {
+    evaluated: boolean;
+    /** Writer identity as published by the controller's own summary line. */
+    isWriter: boolean;
+    leaseMode: LeaseMode | null;
+    /** `summaryLines()` — the spec-mandated observable (writer identity + per-provider row). */
+    summary: string[];
+    providers: Record<string, MultiprocProviderView>;
+    /** On-disk shared values read back through the production reader. */
+    limits: Record<string, number | null>;
+    /** `settings.override` payloads observed since the previous reply. */
+    settingsOverrides: Array<{ path: string; value: unknown }>;
+    /** Warnings emitted since the previous reply. */
+    warnings: string[];
+}
+
+/** Successful reply; `tick` is present on tick replies, `recorded` on failure replies. */
+export interface MultiprocOkReply {
+    id: number;
+    ok: true;
+    tick?: MultiprocTickPayload;
+    recorded?: boolean;
+}
+
+export type MultiprocReply = MultiprocOkReply | { id: number; ok: false; error: string };
+
+// ── state ───────────────────────────────────────────────────────────────────
+
+const WRITER_MARKER = '本进程为写者';
+
+/** The file-lease `O_EXCL` create always fails → `createLease` degrades to read-only. */
+const brokenAtomicFs: WriterFileSystemPort = {
+    ...nodeWriterFileSystem,
+    openExclusive(path: string): void {
+        const err = new Error(`EOPNOTSUPP: atomic create unsupported (${path})`) as NodeJS.ErrnoException;
+        err.code = 'EOPNOTSUPP';
+        throw err;
+    },
+};
+
+const describe = (err: unknown): string => (err instanceof Error ? err.message : String(err));
+
+let controller: ConcurrencyController | null = null;
+let redis: RedisClient | null = null;
+let clock = 0;
+let settingsOverrides: Array<{ path: string; value: unknown }> = [];
+let warnings: string[] = [];
+
+const warn = (text: string): void => {
+    warnings.push(text);
+};
+
+/**
+ * Synthetic cumulative `model_perf` counters (empty until a `perf` command
+ * grows them). An idle worker therefore carries no observation sample — the
+ * exact state the R1 freeze is defined on.
+ */
+let perfRows: ModelPerfRow[] = [];
+
+/**
+ * Grow one provider's cumulative counters by one step. Cumulative on purpose:
+ * the controller diffs consecutive reads, so every `perf` command turns the
+ * next evaluation window into a window with a generation sample.
+ */
+function growPerf(command: MultiprocPerfCommand): void {
+    const modelKey = `${command.provider}/model`;
+    const previous = perfRows.find((row) => row.modelKey === modelKey);
+    const grown: ModelPerfRow = {
+        modelKey,
+        samples: (previous?.samples ?? 0) + (command.samples ?? 10),
+        outputTokens: (previous?.outputTokens ?? 0) + (command.outputTokens ?? 1_000),
+        genMs: (previous?.genMs ?? 0) + (command.genMs ?? 1_000),
+    };
+    perfRows = [...perfRows.filter((row) => row.modelKey !== modelKey), grown];
+}
+
+const now = (): number => clock;
+
+function reply(payload: MultiprocReply): void {
+    process.stdout.write(`${JSON.stringify(payload)}\n`);
+}
+
+function required(): ConcurrencyController {
+    if (!controller) throw new Error('worker not initialized: send `init` first');
+    return controller;
+}
+
+// ── commands ────────────────────────────────────────────────────────────────
+
+function init(command: MultiprocInitCommand): void {
+    if (controller) throw new Error('worker already initialized');
+    clock = command.startAt;
+    if (command.redisPort !== null) {
+        redis = new RedisClient({
+            host: '127.0.0.1',
+            port: command.redisPort,
+            connectTimeoutMs: 500,
+            commandTimeoutMs: 5_000,
+        });
+    }
+    const settings: SettingsPort = {
+        override: (path, value) => {
+            settingsOverrides.push({ path, value });
+        },
+    };
+    const leaseFactory: ConcurrencyLeaseFactory | undefined = command.brokenFileLease
+        ? (args: ConcurrencyLeaseArgs) =>
+              createLease({
+                  redis: args.redis,
+                  lockPath: args.lockPath,
+                  owner: args.owner,
+                  ttlMs: args.ttlMs,
+                  now: args.now,
+                  fileSystem: brokenAtomicFs,
+                  warn,
+              })
+        : undefined;
+    controller = createConcurrencyController({
+        configPath: command.configPath,
+        managedProviders: command.providers,
+        tuner: { ...TUNER_DEFAULTS, ...(command.tuner ?? {}) },
+        now,
+        fs: createNodeFsPort(),
+        owner: command.owner,
+        logDir: command.logDir,
+        modelPerf: { readModelPerf: () => perfRows },
+        redis,
+        settings,
+        warn,
+        ...(command.ceilingByProvider === undefined ? {} : { ceilingByProvider: command.ceilingByProvider }),
+        ...(command.leaseTtlMs === undefined ? {} : { leaseTtlMs: command.leaseTtlMs }),
+        ...(command.writeMinIntervalMs === undefined ? {} : { writeMinIntervalMs: command.writeMinIntervalMs }),
+        ...(leaseFactory === undefined ? {} : { leaseFactory }),
+    });
+}
+
+async function tick(command: MultiprocTickCommand): Promise<MultiprocTickPayload> {
+    const loop = required();
+    clock += command.advanceMs;
+    const result = await loop.onTick();
+
+    const providers: Record<string, MultiprocProviderView> = {};
+    for (const [provider, view] of Object.entries(result.providers)) {
+        const state = loop.stateFor(provider);
+        providers[provider] = {
+            limit: view.limit,
+            initial: state?.initial ?? view.limit,
+            floor: state?.floor ?? 1,
+            ceiling: state?.ceiling ?? view.limit,
+            cleanWindows: state?.cleanWindows ?? 0,
+            badStreak: state?.badStreak ?? 0,
+            cooldownUntil: state?.cooldownUntil ?? 0,
+            badKind: view.badKind,
+            increased: view.increased,
+            decreased: view.decreased,
+            queueSuppressed: view.queueSuppressed,
+            cooldownActive: view.cooldownActive,
+            rateLimitCount: view.rateLimitCount,
+            timeoutCount: view.timeoutCount,
+            written: view.written,
+        };
+    }
+
+    const limits: Record<string, number | null> = {};
+    for (const row of readManagedLimits(loop.configPath, loop.providers)) limits[row.provider] = row.value;
+
+    const summary = loop.summaryLines();
+    const payload: MultiprocTickPayload = {
+        evaluated: result.evaluated,
+        isWriter: summary.some((line) => line.includes(WRITER_MARKER)),
+        leaseMode: loop.leaseMode(),
+        summary,
+        providers,
+        limits,
+        settingsOverrides,
+        warnings,
+    };
+    settingsOverrides = [];
+    warnings = [];
+    return payload;
+}
+
+async function handle(command: MultiprocCommand): Promise<MultiprocReply> {
+    switch (command.cmd) {
+        case 'init':
+            init(command);
+            return { id: command.id, ok: true };
+        case 'tick':
+            return { id: command.id, ok: true, tick: await tick(command) };
+        case 'failure': {
+            const recorded = required().noteExplicitFailure(command.provider, command.text);
+            return { id: command.id, ok: true, recorded };
+        }
+        case 'perf':
+            required();
+            growPerf(command);
+            return { id: command.id, ok: true };
+    }
+}
+
+// ── stdin loop ──────────────────────────────────────────────────────────────
+
+let buffered = '';
+let chain: Promise<void> = Promise.resolve();
+
+const dispatch = (line: string): void => {
+    chain = chain.then(async () => {
+        let command: MultiprocCommand;
+        try {
+            command = JSON.parse(line) as MultiprocCommand;
+        } catch (err) {
+            reply({ id: -1, ok: false, error: `bad command JSON: ${describe(err)}` });
+            return;
+        }
+        try {
+            reply(await handle(command));
+        } catch (err) {
+            reply({ id: command.id, ok: false, error: describe(err) });
+        }
+    });
+};
+
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', (chunk: string) => {
+    buffered += chunk;
+    let index: number;
+    while ((index = buffered.indexOf('\n')) >= 0) {
+        const line = buffered.slice(0, index);
+        buffered = buffered.slice(index + 1);
+        if (line.trim().length > 0) dispatch(line);
+    }
+});
+process.stdin.on('end', () => {
+    void chain.then(() => process.exit(0));
+});