@oxygen-agent/cli 1.1014.135 → 1.1014.618

package/node_modules/@oxygen/shared/dist/diagnostic-spool.js

@@ -0,0 +1,348 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, open, readFile, readdir, rename, stat, unlink, utimes } from "node:fs/promises";
+import { join } from "node:path";
+import { DIAGNOSTIC_MAX_BYTES, diagnosticId } from "./diagnostics.js";
+import { acquireDiagnosticStorageLock } from "./diagnostic-storage-lock.js";
+import { diagnosticRole, StorageFailure, storageFailureFields, storageStage } from "./diagnostic-storage-failure.js";
+import { log } from "./log.js";
+import { recordTelemetryCounter } from "./telemetry.js";
+const admissions = new Map();
+const RECEIPT_MAX_BYTES = 512;
+/** Admit durable content plus bounded receipt bytes; acknowledge only after publication. */
+export async function admitDiagnosticSpool(directory, record, compressed) {
+    await storageStage("mkdir", () => mkdir(directory, { recursive: true, mode: 0o700 }));
+    const path = await admitDiagnostic(directory, record, { compressed });
+    return async () => { await unlink(path).catch(() => undefined); };
+}
+/** Receipt writes share the admission owner's protected byte count and retry reservation. */
+export async function saveDiagnosticUploadProgress(directory, record, progress, reservation) {
+    await admitDiagnostic(directory, record, { compressed: Buffer.alloc(0) }, progress, reservation);
+}
+/** Coalesce local contenders; the kernel mutex still owns cross-process admission. */
+function admitDiagnostic(directory, record, encoded, progress, reservation) {
+    return new Promise((resolve, reject) => {
+        const pending = admissions.get(directory);
+        if (progress && pending?.used !== undefined) {
+            // A retry must not queue behind a new writer's stalled fsync (#5403).
+            // Reserve against the active owner's protected count synchronously; it
+            // keeps the kernel admission lock until every such write has settled.
+            const required = reservation ? 0 : RECEIPT_MAX_BYTES;
+            if (pending.used + required > DIAGNOSTIC_MAX_BYTES) {
+                reject(new StorageFailure("capacity", { code: "spool_full" }));
+                return;
+            }
+            pending.used += required;
+            const write = writeUploadProgress(directory, record, progress, reservation);
+            pending.progressWrites.add(write);
+            void write.then(() => resolve(join(directory, `${record.id}.receipt.json`)), reject).finally(() => { pending.progressWrites.delete(write); });
+            return;
+        }
+        const admission = { record, encoded, ...(progress ? { progress } : {}), ...(reservation ? { reservation } : {}), enqueuedAt: performance.now(), resolve, reject };
+        if (pending)
+            pending.pending.push(admission);
+        else {
+            const queue = { pending: [admission], progressWrites: new Set() };
+            admissions.set(directory, queue);
+            queueMicrotask(() => { void drainAdmissions(directory, queue); });
+        }
+    });
+}
+async function drainAdmissions(directory, queue) {
+    try {
+        while (queue.pending.length) {
+            const batch = queue.pending.splice(0, 32);
+            let release = null;
+            try {
+                // Bound actual cross-process contention, not local queue/inspection time (#5810).
+                release = await storageStage("lock", () => acquireDiagnosticStorageLock(join(directory, ".admission.lock"), 5_000));
+                if (!release)
+                    throw new StorageFailure("lock", { code: "lock_timeout" });
+                const started = performance.now();
+                const reservedOnly = batch.every(admission => admission.progress && admission.reservation);
+                let files = queue.files ?? [];
+                if (!reservedOnly) {
+                    const inspected = await storageStage("inspect", () => inspectDiagnosticSpool(directory));
+                    queue.used = inspected.used;
+                    queue.files = files = inspected.files;
+                }
+                const scanMs = performance.now() - started;
+                // Captures arriving during inspection share its protected byte count,
+                // avoiding another scan while keeping the 32-record batch bound.
+                if (!reservedOnly)
+                    batch.push(...queue.pending.splice(0, 32 - batch.length));
+                for (const admission of batch) {
+                    try {
+                        // Include an upload receipt and its atomic replacement in admission.
+                        const required = admission.reservation ? 0 : admission.progress ? RECEIPT_MAX_BYTES : admission.encoded.compressed.length + 2 * RECEIPT_MAX_BYTES;
+                        if (required && (queue.used === undefined || queue.used + required > DIAGNOSTIC_MAX_BYTES))
+                            throw new StorageFailure("capacity", { code: "spool_full" });
+                        // Reserve conservatively even if cleanup of a failed partial write fails.
+                        if (queue.used !== undefined)
+                            queue.used += required;
+                        if (admission.progress) {
+                            await writeUploadProgress(directory, admission.record, admission.progress, admission.reservation);
+                            admission.resolve(join(directory, `${admission.record.id}.receipt.json`));
+                        }
+                        else
+                            admission.resolve(await writeDiagnosticSpool(directory, admission.record.id, admission.encoded.compressed));
+                    }
+                    catch (error) {
+                        admission.reject(error);
+                    }
+                }
+                if (!reservedOnly)
+                    log("info", "diagnostics.spool_admitted", { queue_role: diagnosticRole(), batch_records: batch.length, spool_files: files.length,
+                        spool_bytes: queue.used, scan_ms: scanMs, admission_wait_ms: performance.now() - batch[0].enqueuedAt });
+            }
+            catch (error) {
+                for (const admission of batch)
+                    admission.reject(error);
+            }
+            finally {
+                delete queue.used;
+                await Promise.allSettled([...queue.progressWrites]);
+                await release?.();
+            }
+        }
+    }
+    catch (error) {
+        for (const admission of queue.pending.splice(0))
+            admission.reject(error);
+    }
+    finally {
+        admissions.delete(directory);
+    }
+}
+async function writeDiagnosticSpool(directory, id, compressed) {
+    const path = join(directory, `${id}.json.gz`);
+    const temporary = join(directory, `${id}.${randomUUID()}.tmp`);
+    let handle;
+    try {
+        handle = await storageStage("open", () => open(temporary, "wx", 0o600));
+        await storageStage("write", () => handle.writeFile(compressed));
+        await storageStage("sync", () => handle.sync());
+        await handle.close();
+        handle = undefined;
+        await storageStage("rename", () => rename(temporary, path));
+        return path;
+    }
+    finally {
+        try {
+            await handle?.close();
+        }
+        finally {
+            await unlink(temporary).catch(() => undefined);
+        }
+    }
+}
+/** A fixed set of stripes bounds SQLite lock files while coordinating direct and retry owners. */
+export async function lockDiagnosticUpload(directory, id, waitMs = 0) {
+    const release = await storageStage("lock", () => acquireDiagnosticStorageLock(join(directory, `.upload-${id.slice(0, 2).toLowerCase()}.lock`), waitMs));
+    if (!release)
+        throw new StorageFailure("lock", { code: "lock_timeout" });
+    return release;
+}
+export async function readUploadProgress(directory, id, storeFingerprint) {
+    const path = join(directory, `${id}.receipt.json`);
+    const info = await stat(path).catch(error => { if (error?.code === "ENOENT")
+        return null; throw error; });
+    if (!info)
+        return null;
+    if (info.size > RECEIPT_MAX_BYTES)
+        throw new Error("Diagnostic upload receipt exceeds its bound");
+    const bytes = await readFile(path).catch(error => { if (error?.code === "ENOENT")
+        return null; throw error; });
+    if (!bytes)
+        return null;
+    if (bytes.length > RECEIPT_MAX_BYTES)
+        throw new Error("Diagnostic upload receipt exceeds its bound");
+    const value = JSON.parse(bytes.toString("utf8"));
+    if (!value || !/^[0-9a-f]{64}$/.test(value.sha256) || value.store !== storeFingerprint || typeof value.published !== "boolean")
+        throw new Error("Diagnostic upload receipt integrity mismatch");
+    return value;
+}
+async function writeUploadProgress(directory, record, progress, reservation) {
+    const bytes = Buffer.from(JSON.stringify(progress));
+    if (bytes.length > RECEIPT_MAX_BYTES)
+        throw new Error("Diagnostic upload receipt exceeds its bound");
+    if (reservation) {
+        // Caller owns the admission mutex. Exchange already-counted bytes for the
+        // new atomic receipt; concurrent inspections cannot miss a rename or grow.
+        const quota = await open(reservation.path, "r+");
+        try {
+            const info = await quota.stat();
+            if (!info.isFile() || info.size < RECEIPT_MAX_BYTES || info.size > 40 * RECEIPT_MAX_BYTES || info.size % RECEIPT_MAX_BYTES)
+                throw new Error("Diagnostic receipt reservation exhausted");
+            await quota.truncate(info.size - RECEIPT_MAX_BYTES);
+            await quota.sync();
+            await utimes(reservation.path, info.atime, info.mtime);
+        }
+        finally {
+            await quota.close();
+        }
+    }
+    const path = join(directory, `${record.id}.receipt.json`);
+    const temporary = join(directory, `${record.id}.${randomUUID()}.tmp`);
+    let handle;
+    try {
+        handle = await open(temporary, "wx", 0o600);
+        await handle.writeFile(bytes);
+        await handle.sync();
+        await handle.close();
+        handle = undefined;
+        // Retry progress does not renew the captured record's 72-hour lifetime.
+        const captured = new Date(record.started_at);
+        await utimes(temporary, captured, captured);
+        await rename(temporary, path);
+        const directoryHandle = await open(directory, "r");
+        try {
+            await directoryHandle.sync();
+        }
+        finally {
+            await directoryHandle.close();
+        }
+    }
+    finally {
+        await handle?.close();
+        await unlink(temporary).catch(() => undefined);
+    }
+}
+/** Caller owns the admission mutex, directly or through an active local writer. */
+async function reserveRetryBatch(directory, files, used, local) {
+    if (!files.length)
+        return { files };
+    const count = Math.min(files.length, Math.floor((DIAGNOSTIC_MAX_BYTES - used) / (2 * RECEIPT_MAX_BYTES)));
+    if (count < 1)
+        throw new StorageFailure("capacity", { code: "spool_full" });
+    const bytes = count * 2 * RECEIPT_MAX_BYTES;
+    // This fixed metadata prefix cannot be a UUID content temp. Its ordinary
+    // .tmp shape remains visible to older admission owners during a rollout.
+    const reservation = { path: join(directory, `f.${randomUUID()}.tmp`) };
+    if (local)
+        local.used = used + bytes;
+    const write = storageStage("receipt", async () => {
+        let handle;
+        try {
+            handle = await open(reservation.path, "wx", 0o600);
+            await handle.writeFile(Buffer.alloc(bytes));
+            await handle.sync();
+            await handle.close();
+            handle = undefined;
+            const parent = await open(directory, "r");
+            try {
+                await parent.sync();
+            }
+            finally {
+                await parent.close();
+            }
+        }
+        catch (error) {
+            await unlink(reservation.path).catch(() => undefined);
+            throw error;
+        }
+        finally {
+            await handle?.close();
+        }
+    });
+    local?.progressWrites.add(write);
+    try {
+        await write;
+    }
+    finally {
+        local?.progressWrites.delete(write);
+    }
+    return { files: files.slice(0, count), reservation };
+}
+/** Replay retained bytes under their stripe; publication owns the upload acknowledgment. */
+export async function retryDiagnosticSpool(directory, storeFingerprint, upload) {
+    const release = await storageStage("lock", () => acquireDiagnosticStorageLock(join(directory, ".upload.lock")));
+    if (!release)
+        return;
+    let reservation;
+    try {
+        // Idle workers enforce the same expiry as new admissions. The admission
+        // lock protects cleanup against a writer replacing an old record path.
+        let files;
+        const local = admissions.get(directory);
+        if (local) {
+            // Reuse the active writer's protected inspection. Waiting for local idle
+            // can starve retained evidence indefinitely under sustained capture.
+            const retained = local.files?.slice(0, 20) ?? [];
+            ({ files, reservation } = local.used === undefined ? { files: [] } : await reserveRetryBatch(directory, retained, local.used, local));
+        }
+        else {
+            const admission = await storageStage("lock", () => acquireDiagnosticStorageLock(join(directory, ".admission.lock")));
+            if (!admission)
+                return;
+            try {
+                const inspected = await storageStage("inspect", () => inspectDiagnosticSpool(directory));
+                ({ files, reservation } = await reserveRetryBatch(directory, inspected.files.slice(0, 20), inspected.used));
+            }
+            finally {
+                await admission();
+            }
+        }
+        for (const name of files) {
+            const path = join(directory, name);
+            try {
+                const release = await lockDiagnosticUpload(directory, name.slice(0, -8));
+                try {
+                    const compressed = await storageStage("read", () => readFile(path));
+                    await upload(compressed, reservation);
+                    await unlink(path).catch(() => undefined);
+                }
+                finally {
+                    await release();
+                }
+            }
+            catch (error) {
+                // A completed owner can remove the path after this sweep's snapshot.
+                // The private acknowledgement proves this is success, rather than loss.
+                if (error instanceof StorageFailure && error.stage === "read" && error.failure?.code === "ENOENT") {
+                    const progress = await readUploadProgress(directory, name.slice(0, -8), storeFingerprint).then(value => value?.published === true).catch(() => false);
+                    if (progress)
+                        continue;
+                }
+                log("warn", "diagnostics.spool_upload_deferred", { diagnostic_id: name.slice(0, -8), ...storageFailureFields(error) });
+            }
+        }
+    }
+    finally {
+        if (reservation)
+            await unlink(reservation.path).catch(() => undefined);
+        await release();
+    }
+}
+/** Caller holds admission lock; count retained bytes and expire actual filenames. */
+async function inspectDiagnosticSpool(directory) {
+    let used = 0;
+    const retained = [];
+    const names = (await readdir(directory)).filter(name => /^[0-9a-f-]+\.(?:json\.gz|receipt\.json)$/.test(name) || /^[0-9a-f-]+(?:\.[0-9a-f-]+)?\.tmp$/.test(name));
+    for (let offset = 0; offset < names.length; offset += 64) {
+        const inspected = await Promise.all(names.slice(offset, offset + 64).map(async (name) => {
+            const path = join(directory, name);
+            const info = await stat(path).catch(error => { if (error?.code === "ENOENT")
+                return null; throw error; });
+            return { name, path, info };
+        }));
+        for (const { name, path, info } of inspected) {
+            const record = /^[0-9a-f-]+\.json\.gz$/.test(name);
+            if (!info)
+                continue;
+            if (Date.now() - info.mtimeMs >= 72 * 60 * 60 * 1000) {
+                const removed = await unlink(path).then(() => true).catch(error => { if (error?.code === "ENOENT")
+                    return false; throw error; });
+                if (removed && !name.endsWith(".receipt.json") && !/^f\.[0-9a-f-]+\.tmp$/.test(name)) {
+                    log("warn", "diagnostics.capture_lost", { reason: "spool_expired", ...(record && diagnosticId(name.slice(0, -8)) ? { diagnostic_id: name.slice(0, -8) } : {}), bytes: info.size, queue_role: diagnosticRole() });
+                    recordTelemetryCounter("oxygen.diagnostics.lost", 1, { reason: "spool_expired" });
+                }
+            }
+            else {
+                used += info.size;
+                if (record)
+                    retained.push({ name, mtimeMs: info.mtimeMs });
+            }
+        }
+    }
+    return { used, files: retained.sort((a, b) => a.mtimeMs - b.mtimeMs || a.name.localeCompare(b.name)).map(file => file.name) };
+}

package/node_modules/@oxygen/shared/dist/diagnostic-storage-failure.d.ts

@@ -0,0 +1,10 @@
+export type StorageStage = "encode" | "mkdir" | "lock" | "inspect" | "capacity" | "open" | "write" | "sync" | "rename" | "read" | "decode" | "upload" | "receipt" | "observer";
+export declare class StorageFailure extends Error {
+    readonly stage: StorageStage;
+    readonly failure: unknown;
+    constructor(stage: StorageStage, failure: unknown);
+}
+export declare function storageStage<T>(stage: StorageStage, action: () => Promise<T>): Promise<T>;
+/** Safe storage metadata shared by capture and the chunk transport; never upstream prose. */
+export declare function storageFailureFields(error: unknown, fallbackStage?: StorageStage): Record<string, string | number | null>;
+export declare function diagnosticRole(): string;

package/node_modules/@oxygen/shared/dist/diagnostic-storage-failure.js

@@ -0,0 +1,42 @@
+export class StorageFailure extends Error {
+    stage;
+    failure;
+    constructor(stage, failure) {
+        super("Diagnostic storage failed");
+        this.stage = stage;
+        this.failure = failure;
+    }
+}
+export async function storageStage(stage, action) {
+    try {
+        return await action();
+    }
+    catch (error) {
+        throw error instanceof StorageFailure ? error : new StorageFailure(stage, error);
+    }
+}
+/** Safe storage metadata shared by capture and the chunk transport; never upstream prose. */
+export function storageFailureFields(error, fallbackStage = "encode") {
+    const stage = error instanceof StorageFailure ? error.stage : fallbackStage;
+    const failure = error instanceof StorageFailure ? error.failure : error;
+    const value = failure && typeof failure === "object" ? failure : {};
+    const code = typeof value.code === "string" && /^(?:E[A-Z0-9_]{1,30}|SQLITE_[A-Z_]{1,30}|spool_full|lock_timeout)$/.test(value.code) ? value.code : null;
+    const sqlite = typeof value.errcode === "number" ? value.errcode : null;
+    const status = value.$metadata?.httpStatusCode ?? value.status;
+    const httpStatus = typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599 ? status : null;
+    const timeout = [value.name, value.cause?.name].some(name => typeof name === "string" && ["AbortError", "TimeoutError", "RequestTimeout"].includes(name))
+        || [value.code, value.cause?.code].some(code => typeof code === "string" && ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ABORT_ERR"].includes(code));
+    const cause = code === "lock_timeout" ? "lock_timeout" : code === "spool_full" ? "spool_limit"
+        : ["EACCES", "EPERM", "EROFS"].includes(code ?? "") || (sqlite !== null && [3, 8, 23].includes(sqlite & 255)) ? "permission"
+            : ["ENOSPC", "EDQUOT"].includes(code ?? "") || (sqlite !== null && (sqlite & 255) === 13) ? "filesystem_capacity"
+                : httpStatus === 507 && stage === "upload" ? "object_storage_capacity" : httpStatus === 429 ? "rate_limit"
+                    : timeout ? "network_timeout" : [code, value.cause?.code].some(code => typeof code === "string" && ["ECONNRESET", "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", "EPIPE", "ENOTFOUND", "EAI_AGAIN"].includes(code)) ? "network"
+                        : httpStatus !== null && httpStatus >= 500 ? "http_server" : httpStatus !== null && httpStatus >= 400 ? "http_client" : "other";
+    return { cause, stage, error_code: code, sqlite_code: sqlite, errno: typeof value.errno === "number" ? value.errno : null,
+        http_status: httpStatus, queue_role: diagnosticRole() };
+}
+export function diagnosticRole() {
+    const role = process.env.OXYGEN_WORKER_QUEUE_ROLE;
+    return role && ["relay", "bulk", "maintenance", "interactive", "provider", "outbound"].includes(role) ? role
+        : process.env.OXYGEN_PROCESS_ROLE === "worker" ? "legacy" : "web";
+}

package/node_modules/@oxygen/shared/dist/diagnostic-storage.d.ts

@@ -1,5 +1,6 @@
 import { S3Client } from "@aws-sdk/client-s3";
 import { type DiagnosticRecord } from "./diagnostics.js";
+export { storageFailureFields as diagnosticStorageFailureFields } from "./diagnostic-storage-failure.js";
 export type DiagnosticStoredReceipt = {
     key: string;
     sha256: string;
@@ -28,4 +29,3 @@ export declare function storeDiagnostic(record: DiagnosticRecord): Promise<Diagn
 export declare function flushDiagnosticSpool(): Promise<void>;
 /** A metadata-only fallback for an unsupported value never claims full capture. */
 export declare function partialDiagnostic(record: DiagnosticRecord, reason: string): DiagnosticRecord;
-export {};