@playcraft/cli 0.1.85-beta → 0.1.86

package/dist/workspace-runtime/codex/model-capture.js

@@ -1,675 +0,0 @@
-import { createHash, randomUUID } from 'node:crypto';
-import { mkdir, writeFile, readFile, readdir, rm, rename, lstat, realpath } from 'node:fs/promises';
-import { join, resolve, relative, isAbsolute } from 'node:path';
-import { z } from 'zod';
-import { platformEndpoint, } from '@playcraft/common/cloud-workspace/runtime';
-import { MODEL_CAPTURE_CHUNK_BYTES, } from '@playcraft/common/model-proxy';
-import { isCaptureIsolationAvailable } from './capture-isolation.js';
-const QUEUE_BYTES = 8 * 1024 * 1024;
-const SPOOL_BYTES = 512 * 1024 * 1024;
-const MAX_CAPTURE_CALLS = 128;
-const manifestSchema = z.object({
-    version: z.literal(1),
-    identity: z.object({
-        modelCallId: z.string().uuid(),
-        workspaceId: z.string().uuid(),
-        sessionId: z.string().min(1),
-        clientRequestId: z.string().min(1),
-        directoryName: z.string().min(1).optional(),
-        runtimeId: z.string().min(1),
-        generation: z.number().int().min(0),
-        runtimeEpoch: z.number().int().min(0),
-        configId: z.string().uuid(),
-        captureCallRowId: z.string().uuid().optional(),
-        expiresAt: z.string().datetime(),
-    }),
-    artifacts: z
-        .array(z.object({
-        stage: z.enum(['codex_request', 'codex_response']),
-        contentType: z.string().max(256),
-        representation: z.enum(['raw-http-body', 'decoded_body']),
-        parts: z.number().int().min(0).max(128),
-        rawBytes: z.number().int().min(0).max(SPOOL_BYTES),
-        end: z
-            .object({
-            sourceEnd: z.enum([
-                'eof',
-                'aborted',
-                'cancelled_by_failover',
-                'process_lost',
-                'limit_exceeded',
-                'io_error',
-                'not_started',
-            ]),
-            protocolTerminal: z.enum(['observed', 'absent', 'not_applicable', 'unknown']),
-            downstreamFinished: z.boolean().optional(),
-            dispatchObserved: z.boolean().optional(),
-            responseHeadersObserved: z.boolean().optional(),
-            responseStatus: z.number().int().min(100).max(599).optional(),
-            upstreamObservedBytes: z.number().int().min(0).optional(),
-            downstreamSubmittedBytes: z.number().int().min(0).optional(),
-        })
-            .optional(),
-    }))
-        .max(2),
-});
-function within(parent, child) {
-    const path = relative(parent, child);
-    return !path || (!path.startsWith('..') && !isAbsolute(path));
-}
-let queuedBytes = 0;
-let uploadCount = 0;
-const uploadWaiters = [];
-async function uploadSlot(operation) {
-    if (uploadCount >= 2)
-        await new Promise((resolve) => uploadWaiters.push(resolve));
-    else
-        uploadCount++;
-    try {
-        return await operation();
-    }
-    finally {
-        const next = uploadWaiters.shift();
-        if (next)
-            next();
-        else
-            uploadCount--;
-    }
-}
-/** Captures are control-process state. Only immutable scope and raw parts reach disk. */
-export class WorkspaceModelCapture {
-    state;
-    root;
-    diskBytes = 0;
-    closed = false;
-    ready = false;
-    calls = new Set();
-    constructor(state) {
-        this.state = state;
-        this.root = join(state.config.stateDir, 'model-captures');
-    }
-    async start() {
-        if (!isCaptureIsolationAvailable())
-            return;
-        try {
-            await mkdir(this.root, { recursive: true, mode: 0o700 }); // private
-            const info = await lstat(this.root);
-            const rootPath = await realpath(this.root);
-            const workspacePath = await realpath(this.state.config.workspaceRoot).catch(() => resolve(this.state.config.workspaceRoot));
-            const codexPath = await realpath(process.env.PLAYCRAFT_CODEX_HOME || '/var/lib/playcraft/codex').catch(() => resolve(process.env.PLAYCRAFT_CODEX_HOME || '/var/lib/playcraft/codex'));
-            if (!info.isDirectory() ||
-                info.isSymbolicLink() ||
-                (info.mode & 0o077) !== 0 ||
-                (process.getuid && info.uid !== process.getuid()) ||
-                within(workspacePath, rootPath) ||
-                within(codexPath, rootPath))
-                return;
-            this.ready = true;
-            for (const name of await readdir(this.root)) {
-                if (!z.string().uuid().safeParse(name).success)
-                    continue;
-                const directory = join(this.root, name);
-                try {
-                    const entry = await lstat(directory);
-                    if (!entry.isDirectory() ||
-                        entry.isSymbolicLink() ||
-                        entry.uid !== info.uid ||
-                        entry.mode & 0o077)
-                        throw new Error('capture_private_directory_required');
-                    const manifest = manifestSchema.parse(JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8')));
-                    if (manifest.version !== 1 ||
-                        manifest.identity.modelCallId !== name ||
-                        manifest.identity.runtimeId !== this.state.config.runtimeId ||
-                        manifest.identity.generation !== this.state.config.generation ||
-                        manifest.identity.runtimeEpoch !== this.state.runtimeEpoch ||
-                        Date.parse(manifest.identity.expiresAt) <= Date.now() ||
-                        !manifest.identity.captureCallRowId ||
-                        this.calls.size >= MAX_CAPTURE_CALLS) {
-                        await rm(directory, { recursive: true, force: true });
-                        continue;
-                    }
-                    let existingBytes = 0;
-                    const stages = new Set();
-                    for (const artifact of manifest.artifacts) {
-                        if (stages.has(artifact.stage))
-                            throw new Error('capture_duplicate_artifact');
-                        stages.add(artifact.stage);
-                        let actualBytes = 0;
-                        for (let partNo = 0; partNo < artifact.parts; partNo++) {
-                            const part = await lstat(join(directory, `${artifact.stage}.${partNo}.part`));
-                            if (!part.isFile() ||
-                                part.isSymbolicLink() ||
-                                part.uid !== info.uid ||
-                                part.mode & 0o077 ||
-                                part.size > MODEL_CAPTURE_CHUNK_BYTES)
-                                throw new Error('capture_private_part_required');
-                            actualBytes += part.size;
-                        }
-                        if (actualBytes !== artifact.rawBytes)
-                            throw new Error('capture_part_manifest_mismatch');
-                        existingBytes += actualBytes;
-                        if (!artifact.end)
-                            artifact.end = {
-                                sourceEnd: 'process_lost',
-                                protocolTerminal: 'unknown',
-                                ...(artifact.stage === 'codex_response' ? { downstreamFinished: false } : {}),
-                            };
-                    }
-                    if (this.diskBytes + existingBytes > SPOOL_BYTES)
-                        throw new Error('capture_spool_limit');
-                    this.diskBytes += existingBytes;
-                    const call = new CaptureCall(this, directory, manifest);
-                    this.calls.add(call);
-                    void call.upload();
-                }
-                catch {
-                    await rm(directory, { recursive: true, force: true }).catch(() => { });
-                }
-            }
-        }
-        catch {
-            /* Capture setup failure cannot prevent the loopback broker from starting. */
-        }
-    }
-    begin(modelCallId, credential) {
-        const policy = credential.claims.capturePolicy;
-        const origin = credential.claims.traceOrigin;
-        if (this.closed ||
-            !this.ready ||
-            this.calls.size >= MAX_CAPTURE_CALLS ||
-            !policy?.enabled ||
-            !origin ||
-            !('directory' in origin) ||
-            Date.parse(policy.expiresAt) <= Date.now())
-            return undefined;
-        const identity = {
-            modelCallId,
-            workspaceId: credential.claims.workspaceId,
-            runtimeId: credential.claims.runtimeId,
-            generation: credential.claims.generation,
-            runtimeEpoch: credential.claims.runtimeEpoch,
-            sessionId: origin.sessionId,
-            directoryName: typeof origin.directory === 'string' ? origin.directory : undefined,
-            clientRequestId: origin.clientRequestId,
-            configId: credential.configId,
-            expiresAt: new Date(Math.min(Date.parse(policy.expiresAt), Date.now() + 3_600_000)).toISOString(),
-        };
-        const call = new CaptureCall(this, join(this.root, modelCallId), { version: 1, identity, artifacts: [] }, credential.token, {
-            request: Math.min(policy.maxRequestBytes, 128 * 1024 * 1024),
-            response: Math.min(policy.maxResponseBytes, 512 * 1024 * 1024),
-        });
-        this.calls.add(call);
-        return call;
-    }
-    reserve(bytes) {
-        if (this.diskBytes + bytes > SPOOL_BYTES)
-            return false;
-        this.diskBytes += bytes;
-        return true;
-    }
-    release(bytes) {
-        this.diskBytes = Math.max(0, this.diskBytes - bytes);
-    }
-    done(call) {
-        this.calls.delete(call);
-    }
-    get runtime() {
-        return this.state;
-    }
-    get supported() {
-        return this.ready && !this.closed;
-    }
-    async close() {
-        this.closed = true;
-        // Only local writes are drained. Network/storage availability never delays shutdown.
-        await Promise.allSettled([...this.calls].map((call) => call.flush()));
-        for (const call of this.calls)
-            call.close();
-    }
-}
-export class CaptureCall {
-    owner;
-    directory;
-    manifest;
-    limits;
-    writes = Promise.resolve();
-    manifests = Promise.resolve();
-    token;
-    tokenExpires = 0;
-    uploading = false;
-    stopped = false;
-    reservedBytes = 0;
-    removing;
-    retry;
-    artifacts = new Map();
-    nextParts = new Map();
-    failedStages = new Set();
-    constructor(owner, directory, manifest, token, limits = { request: 128 * 1024 * 1024, response: 512 * 1024 * 1024 }) {
-        this.owner = owner;
-        this.directory = directory;
-        this.manifest = manifest;
-        this.limits = limits;
-        this.token = token;
-        this.reservedBytes = manifest.artifacts.reduce((sum, artifact) => sum + artifact.rawBytes, 0);
-    }
-    artifact(stage, contentType = 'application/octet-stream', representation = 'raw-http-body') {
-        let artifact = this.artifacts.get(stage);
-        if (!artifact) {
-            const record = { stage, contentType, representation, parts: 0, rawBytes: 0 };
-            this.manifest.artifacts.push(record);
-            artifact = new CaptureArtifact(this, record, stage === 'codex_request' ? this.limits.request : this.limits.response);
-            this.artifacts.set(stage, artifact);
-            this.schedule(async () => {
-                if (this.stopped)
-                    return;
-                await mkdir(this.directory, { recursive: true, mode: 0o700 }); // private
-                await this.persist();
-            });
-        }
-        return artifact;
-    }
-    schedule(operation) {
-        this.writes = this.writes.then(operation).catch(() => {
-            this.stopped = true;
-            // Run after this chain's rejection handler returns, so cleanup cannot await itself.
-            setImmediate(() => {
-                void this.remove();
-            });
-        });
-    }
-    part(record, bytes) {
-        const partNo = this.nextParts.get(record.stage) ?? record.parts;
-        this.nextParts.set(record.stage, partNo + 1);
-        if (!this.owner.reserve(bytes.length)) {
-            throw new Error('capture_spool_limit');
-        }
-        this.reservedBytes += bytes.length;
-        this.schedule(async () => {
-            let written = false;
-            try {
-                if (this.stopped || this.failedStages.has(record.stage)) {
-                    this.release(bytes.length);
-                    return;
-                }
-                await writeFile(this.partPath(record.stage, partNo), bytes, { flag: 'wx', mode: 0o600 }); // private
-                written = true;
-                record.rawBytes += bytes.length;
-                record.parts = partNo + 1;
-                await this.persist();
-            }
-            catch (error) {
-                if (!written)
-                    this.release(bytes.length);
-                this.failedStages.add(record.stage);
-            }
-            finally {
-                queuedBytes -= bytes.length;
-            }
-        });
-    }
-    finish(record, end) {
-        this.schedule(async () => {
-            if (this.stopped)
-                return;
-            record.end = {
-                ...end,
-                sourceEnd: this.failedStages.has(record.stage) ? 'io_error' : end.sourceEnd,
-            };
-            await this.persist();
-        });
-        void this.upload();
-    }
-    flush() {
-        return this.writes;
-    }
-    close() {
-        this.stopped = true;
-        if (this.retry)
-            clearTimeout(this.retry);
-    }
-    get active() {
-        return !this.stopped;
-    }
-    release(bytes) {
-        this.owner.release(bytes);
-        this.reservedBytes = Math.max(0, this.reservedBytes - bytes);
-    }
-    remove() {
-        if (this.removing)
-            return this.removing;
-        this.stopped = true;
-        for (const artifact of this.artifacts.values())
-            artifact.finish({ sourceEnd: 'io_error', protocolTerminal: 'unknown' });
-        if (this.retry)
-            clearTimeout(this.retry);
-        this.removing = (async () => {
-            await this.writes;
-            await this.manifests.catch(() => { });
-            try {
-                await rm(this.directory, { recursive: true, force: true });
-                this.release(this.reservedBytes);
-            }
-            catch {
-                /* Keep quota reserved while files cannot be removed. */
-            }
-            finally {
-                this.owner.done(this);
-            }
-        })();
-        return this.removing;
-    }
-    partPath(stage, part) {
-        return join(this.directory, `${stage}.${part}.part`);
-    }
-    persist() {
-        // Registration and body writes share one manifest writer, including snapshot creation.
-        const next = this.manifests
-            .catch(() => { })
-            .then(async () => {
-            const temporary = join(this.directory, `${randomUUID()}.tmp`);
-            await writeFile(temporary, JSON.stringify(this.manifest), { mode: 0o600 }); // private
-            await rename(temporary, join(this.directory, 'manifest.json'));
-        });
-        this.manifests = next;
-        return next;
-    }
-    async authorize() {
-        if (this.token && (!this.tokenExpires || this.tokenExpires > Date.now() + 10_000))
-            return this.token;
-        const config = this.owner.runtime.config;
-        const response = await fetch(platformEndpoint(config.publicBaseUrl, '/runtime/internal/model-capture-upload-grants'), {
-            method: 'POST',
-            redirect: 'error',
-            signal: AbortSignal.timeout(10_000),
-            headers: {
-                authorization: `Bearer ${config.controlToken}`,
-                'content-type': 'application/json',
-                'playcraft-env': config.environment,
-            },
-            body: JSON.stringify({
-                runtimeId: this.manifest.identity.runtimeId,
-                generation: this.manifest.identity.generation,
-                runtimeEpoch: this.manifest.identity.runtimeEpoch,
-                workspaceId: this.manifest.identity.workspaceId,
-                sessionId: this.manifest.identity.sessionId,
-                directoryName: this.manifest.identity.directoryName,
-                clientRequestId: this.manifest.identity.clientRequestId,
-                modelCallId: this.manifest.identity.modelCallId,
-                captureCallRowId: this.manifest.identity.captureCallRowId,
-            }),
-        });
-        if (!response.ok) {
-            await response.body?.cancel();
-            if ([401, 403, 404, 410].includes(response.status))
-                await this.remove();
-            throw new Error('capture_authority_unavailable');
-        }
-        const grant = z
-            .object({ token: z.string().min(1), expiresAt: z.string().datetime() })
-            .parse(await response.json());
-        this.token = grant.token;
-        this.tokenExpires = Date.parse(grant.expiresAt);
-        return grant.token;
-    }
-    async send(path, init) {
-        const token = await this.authorize();
-        const base = `/api/llm-proxy/workspace-captures/${this.manifest.identity.modelCallId}`;
-        const response = await fetch(platformEndpoint(this.owner.runtime.config.publicBaseUrl, `${base}${path}`), {
-            ...init,
-            redirect: 'error',
-            signal: AbortSignal.timeout(15_000),
-            headers: { ...init.headers, authorization: `Bearer ${token}` },
-        });
-        if (!response.ok) {
-            const terminal = response.status === 410 ||
-                ((response.status === 401 || response.status === 403) && this.tokenExpires > 0);
-            if (response.status === 401 || response.status === 403) {
-                this.token = undefined;
-                this.tokenExpires = 0;
-            }
-            await response.body?.cancel();
-            if (terminal)
-                await this.remove();
-            throw new Error('capture_upload_unavailable');
-        }
-        return response;
-    }
-    verifyArchive(record, sha256, remote) {
-        if (remote.stage !== record.stage ||
-            remote.archiveState !== 'archived' ||
-            remote.rawBytes !== record.rawBytes ||
-            remote.partCount !== record.parts ||
-            remote.sha256 !== sha256 ||
-            Object.entries(record.end ?? {}).some(([key, value]) => remote[key] !== value)) {
-            throw new Error('capture_archive_confirmation_mismatch');
-        }
-    }
-    async upload() {
-        if (this.uploading || this.stopped)
-            return;
-        this.uploading = true;
-        try {
-            await this.writes;
-            if (this.stopped)
-                return;
-            if (Date.parse(this.manifest.identity.expiresAt) <= Date.now()) {
-                await this.remove();
-                return;
-            }
-            await uploadSlot(async () => {
-                if (!this.manifest.identity.captureCallRowId) {
-                    const response = await this.send('/register', {
-                        method: 'POST',
-                        headers: { 'content-type': 'application/json' },
-                        body: JSON.stringify({ configId: this.manifest.identity.configId }),
-                    });
-                    const registration = (await response.json());
-                    if (registration.capability !== 'available' || !registration.call) {
-                        await this.remove();
-                        return;
-                    }
-                    this.manifest.identity.captureCallRowId = registration.call.callRowId;
-                    await this.persist();
-                }
-                const statusResponse = await this.send('/upload-status', { method: 'GET' });
-                const status = (await statusResponse.json());
-                if (status.call?.callRowId !== this.manifest.identity.captureCallRowId ||
-                    status.call.modelCallId !== this.manifest.identity.modelCallId ||
-                    !Array.isArray(status.artifacts)) {
-                    throw new Error('capture_status_identity_mismatch');
-                }
-                for (const record of this.manifest.artifacts) {
-                    if (!record.end)
-                        continue;
-                    const remote = status.artifacts.find((item) => item.stage === record.stage);
-                    const confirmed = new Map((remote?.confirmedParts ?? []).map((part) => [part.partNo, part]));
-                    const hash = createHash('sha256');
-                    for (let partNo = 0; partNo < record.parts; partNo++) {
-                        const bytes = await readFile(this.partPath(record.stage, partNo));
-                        hash.update(bytes);
-                        const sha256 = createHash('sha256').update(bytes).digest('hex');
-                        const part = confirmed.get(partNo);
-                        if (part) {
-                            if (part.sha256 !== sha256 || part.rawBytes !== bytes.length)
-                                throw new Error('capture_part_confirmation_mismatch');
-                            continue;
-                        }
-                        const response = await this.send(`/artifacts/${record.stage}/chunks/${partNo}`, {
-                            method: 'PUT',
-                            headers: {
-                                'content-type': 'application/octet-stream',
-                                'x-capture-sha256': sha256,
-                                'x-capture-content-type': record.contentType,
-                                'x-capture-representation': record.representation,
-                            },
-                            body: bytes,
-                        });
-                        await response.body?.cancel();
-                    }
-                    const sha256 = hash.digest('hex');
-                    if (remote?.archiveState === 'archived') {
-                        this.verifyArchive(record, sha256, remote);
-                        continue;
-                    }
-                    const response = await this.send(`/artifacts/${record.stage}/finalize`, {
-                        method: 'POST',
-                        headers: { 'content-type': 'application/json' },
-                        body: JSON.stringify({
-                            ...record.end,
-                            partCount: record.parts,
-                            rawBytes: record.rawBytes,
-                            sha256,
-                        }),
-                    });
-                    this.verifyArchive(record, sha256, (await response.json()));
-                }
-            });
-            if ((this.manifest.artifacts.length === 2 || this.artifacts.size === 0) &&
-                this.manifest.artifacts.every((item) => item.end)) {
-                await this.remove();
-                return;
-            }
-        }
-        catch {
-            /* Retry only archival transport, never model execution. */
-        }
-        finally {
-            this.uploading = false;
-            if (!this.stopped) {
-                this.retry = setTimeout(() => {
-                    void this.upload();
-                }, 5_000);
-                this.retry.unref();
-            }
-        }
-    }
-}
-export class CaptureArtifact {
-    call;
-    record;
-    limit;
-    pending = [];
-    pendingBytes = 0;
-    receivedBytes = 0;
-    ended = false;
-    failure;
-    observedBytes = 0;
-    submittedBytes = 0;
-    constructor(call, record, limit) {
-        this.call = call;
-        this.record = record;
-        this.limit = limit;
-    }
-    contentType(value) {
-        this.record.contentType = value.slice(0, 256);
-    }
-    upstreamObserved(bytes) {
-        this.observedBytes += bytes;
-    }
-    append(bytes) {
-        if (this.ended || !bytes.byteLength)
-            return;
-        this.submittedBytes += bytes.byteLength;
-        if (this.failure || !this.call.active)
-            return;
-        if (this.receivedBytes + bytes.byteLength > this.limit ||
-            queuedBytes + bytes.byteLength > QUEUE_BYTES) {
-            this.failure = 'limit_exceeded';
-            return;
-        }
-        this.receivedBytes += bytes.byteLength;
-        queuedBytes += bytes.byteLength;
-        let offset = 0;
-        while (offset < bytes.byteLength) {
-            const length = Math.min(MODEL_CAPTURE_CHUNK_BYTES - this.pendingBytes, bytes.byteLength - offset);
-            this.pending.push(Buffer.from(bytes.subarray(offset, offset + length)));
-            this.pendingBytes += length;
-            offset += length;
-            if (this.pendingBytes === MODEL_CAPTURE_CHUNK_BYTES)
-                this.flushPart();
-        }
-    }
-    finish(end) {
-        if (this.ended)
-            return;
-        this.ended = true;
-        if (!this.call.active) {
-            queuedBytes -= this.pendingBytes;
-            this.pending = [];
-            this.pendingBytes = 0;
-            return;
-        }
-        this.flushPart();
-        this.call.finish(this.record, {
-            ...end,
-            sourceEnd: this.failure ?? end.sourceEnd,
-            ...(this.record.stage === 'codex_response'
-                ? {
-                    upstreamObservedBytes: this.observedBytes,
-                    downstreamSubmittedBytes: this.submittedBytes,
-                }
-                : {}),
-        });
-    }
-    flushPart() {
-        if (!this.pendingBytes)
-            return;
-        const bytes = Buffer.concat(this.pending, this.pendingBytes);
-        this.pending = [];
-        this.pendingBytes = 0;
-        try {
-            this.call.part(this.record, bytes);
-        }
-        catch {
-            queuedBytes -= bytes.length;
-            this.failure = 'limit_exceeded';
-        }
-    }
-}
-/** Observes Node's accepted bytes without changing the return value or drain handling. */
-export function captureSubmittedResponse(response, artifact) {
-    const append = (chunk, encoding) => {
-        try {
-            if (typeof chunk === 'string')
-                artifact.append(Buffer.from(chunk, typeof encoding === 'string' ? encoding : undefined));
-            else if (chunk instanceof Uint8Array)
-                artifact.append(chunk);
-        }
-        catch {
-            /* Archival must never turn a successful write into an inference failure. */
-        }
-    };
-    const write = response.write;
-    response.write = function (...args) {
-        const writable = !this.destroyed && !this.writableEnded && !this.socket?.destroyed;
-        const accepted = Reflect.apply(write, this, args);
-        if (writable && !this.destroyed && !this.socket?.destroyed)
-            append(args[0], args[1]);
-        return accepted;
-    };
-    const end = response.end;
-    response.end = function (chunk, encoding, callback) {
-        const writable = !this.destroyed && !this.writableEnded && !this.socket?.destroyed;
-        const accepted = Reflect.apply(end, this, [chunk, encoding, callback]);
-        if (writable && !this.destroyed && !this.socket?.destroyed)
-            append(chunk, encoding);
-        return accepted;
-    };
-    response.once('finish', () => queueMicrotask(() => {
-        artifact.contentType(String(response.getHeader('content-type') ?? 'application/octet-stream'));
-        artifact.finish({
-            sourceEnd: 'eof',
-            protocolTerminal: /event-stream|ndjson/i.test(String(response.getHeader('content-type') ?? ''))
-                ? 'unknown'
-                : 'not_applicable',
-            downstreamFinished: true,
-            responseStatus: response.statusCode,
-        });
-    }));
-    response.once('close', () => queueMicrotask(() => artifact.finish({
-        sourceEnd: response.writableFinished ? 'eof' : 'aborted',
-        protocolTerminal: 'unknown',
-        downstreamFinished: response.writableFinished,
-        responseStatus: response.statusCode,
-    })));
-}