maka-agent 0.2.0-dev.31.20260913 → 0.2.0-dev.32.20260914

package/node_modules/@maka/eval/dist/fleet-simulation.js

@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { pathToFileURL } from 'node:url';
+import { summarizeFleet } from './fleet.js';
+import { FleetCoordinator, MemoryFleetPersistence } from './fleet-store.js';
+import { FleetWorker } from './fleet-worker.js';
+/** Replayable virtual-time fault simulation, exercising the real coordinator and worker loop. */
+export async function simulateFleet(seed) {
+    if (!Number.isSafeInteger(seed) || seed < 0 || seed > 0xffffffff)
+        throw new Error('seed must be uint32');
+    let randomState = seed;
+    const random = () => {
+        randomState = (Math.imul(1664525, randomState) + 1013904223) >>> 0;
+        return randomState;
+    };
+    const spec = {
+        schemaVersion: 'maka.eval.v1',
+        id: 'fleet-simulation',
+        benchmark: { id: 'mock-benchmark', version: '1', config: {} },
+        executor: { kind: 'mock', config: {} },
+        execution: { maxConcurrentTaskGroups: 3 },
+        subjects: ['a', 'b', 'c'].map((id) => ({ id, kind: 'external', credentials: [], config: {} })),
+        tasks: Array.from({ length: 20 }, (_, index) => ({
+            id: `task-${index}`,
+            input: 'mock',
+            config: {},
+        })),
+        repetitions: 2,
+        budget: {},
+        verifier: {},
+    };
+    const policy = {
+        runId: `seed-${seed}`,
+        environmentId: 'mock-manifest-v1',
+        groupCpus: 6,
+        groupMemoryMb: 1024,
+        leaseMs: 12,
+        recoveryGraceMs: 15,
+        retryBackoffMs: 2,
+        maxAttemptsPerCell: 20,
+    };
+    let now = 0;
+    const persistence = new MemoryFleetPersistence();
+    let coordinator = await FleetCoordinator.open({ persistence, spec, policy, now: () => now });
+    const trace = [];
+    let order = 0;
+    const events = [];
+    let crashed = false;
+    const lost = new Set();
+    const committed = new Map();
+    const workers = Array.from({ length: 3 }, (_, index) => {
+        const id = `vm-${index}/boot-1`;
+        const disconnected = () => (now >= 55 && now < 100) || (index === 0 && now >= 8 && now < 38);
+        const transport = {
+            async command(command) {
+                if (disconnected())
+                    throw new SimulatedLinkError();
+                const roll = random();
+                if (now < 240 && roll % 31 === 0) {
+                    trace.push(`${now} ${id} ${command.kind} dropped-before`);
+                    throw new SimulatedLinkError();
+                }
+                const reply = await coordinator.command(command);
+                const once = `${id}/${command.kind}`;
+                // Guaranteed lost claim/report acknowledgements, plus seeded background faults.
+                if (now < 240 &&
+                    ((['claim', 'report'].includes(command.kind) && !lost.has(once)) || roll % 37 === 0)) {
+                    lost.add(once);
+                    trace.push(`${now} ${id} ${command.kind} dropped-after`);
+                    throw new SimulatedLinkError();
+                }
+                return reply;
+            },
+            async putArtifact(bytes) {
+                if (disconnected())
+                    throw new SimulatedLinkError();
+                return coordinator.putArtifact(bytes);
+            },
+        };
+        return new FleetWorker({
+            id,
+            environmentId: policy.environmentId,
+            cpus: 6,
+            memoryMb: 1024,
+            groupSlots: 1,
+        }, transport, (work, emit) => new Promise((resolve) => {
+            trace.push(`${now} ${id} start ${work.assignment.id} cells=${work.cells.length}`);
+            let remaining = work.cells.length;
+            if (!remaining) {
+                resolve();
+                return;
+            }
+            for (const cell of work.cells) {
+                const delay = 2 + (random() % 19);
+                events.push({
+                    at: now + delay,
+                    order: order++,
+                    action: () => {
+                        if (index === 2 && crashed)
+                            return;
+                        const output = simulationOutput();
+                        // Valid outcomes with absent metering must remain selectable.
+                        if (random() % 5 === 0) {
+                            output.report.usage = 'missing';
+                            output.report.result = {
+                                ...output.report.result,
+                                usage: null,
+                                costUsd: null,
+                                status: 'indeterminate',
+                            };
+                        }
+                        emit(cell.attemptId, output);
+                        trace.push(`${now} ${id} completed ${cell.attemptId}`);
+                        if (--remaining === 0)
+                            resolve();
+                    },
+                });
+            }
+        }));
+    });
+    for (now = 0; now < 3000; now++) {
+        if (now === 20) {
+            crashed = true;
+            trace.push(`${now} vm-2 process-crash`);
+        }
+        if (now === 55) {
+            await coordinator.command({ kind: 'pause' });
+            trace.push(`${now} coordinator-offline`);
+        }
+        if (now === 100) {
+            await coordinator.close();
+            coordinator = await FleetCoordinator.open({ persistence, spec, policy, now: () => now });
+            trace.push(`${now} coordinator-restarted`);
+        }
+        events.sort((a, b) => a.at - b.at || a.order - b.order);
+        while (events[0]?.at <= now)
+            events.shift().action();
+        // Drain the finite execute/catch/finally chain, without timers or real sleeps.
+        for (let i = 0; i < 8; i++)
+            await Promise.resolve();
+        for (const [index, worker] of workers.entries()) {
+            if (index === 2 && crashed)
+                continue;
+            try {
+                await worker.poll();
+            }
+            catch (error) {
+                if (!(error instanceof SimulatedLinkError))
+                    throw error;
+            }
+        }
+        await coordinator.command({ kind: 'tick' });
+        const state = await coordinator.snapshot();
+        try {
+            assertFleetInvariants(state);
+            for (const [id, evidence] of committed) {
+                const attempt = state.attempts.find((a) => a.id === id);
+                if (!attempt || JSON.stringify(attempt) !== evidence)
+                    throw new Error('committed result changed or disappeared');
+            }
+            for (const attempt of state.attempts) {
+                if (attempt.disposition === 'selected')
+                    committed.set(attempt.id, JSON.stringify(attempt));
+            }
+        }
+        catch (error) {
+            throw new Error(`simulation invariant failed: seed=${seed} tick=${now}\n${trace.slice(-30).join('\n')}`, { cause: error });
+        }
+        if (summarizeFleet(state).settled)
+            return { seed, elapsedTicks: now, trace, state, summary: summarizeFleet(state) };
+    }
+    throw new Error(`simulation did not settle: seed=${seed}\n${trace.slice(-30).join('\n')}`);
+}
+/** Cross-cutting safety oracle, checked after every simulated tick. */
+export function assertFleetInvariants(state) {
+    const fail = (message) => {
+        throw new Error(`fleet invariant: ${message}`);
+    };
+    const active = state.assignments.filter((a) => a.status === 'active');
+    if (active.length > state.spec.execution.maxConcurrentTaskGroups)
+        fail('global capacity');
+    if (new Set(active.map((a) => a.groupId)).size !== active.length)
+        fail('group ownership');
+    for (const worker of state.workers) {
+        if (active.filter((a) => a.workerId === worker.description.id).length >
+            worker.description.groupSlots)
+            fail('worker capacity');
+    }
+    for (const cellId of state.groups.flatMap((g) => g.cellIds)) {
+        const attempts = state.attempts.filter((a) => a.cellId === cellId);
+        if (attempts.filter((a) => a.disposition === 'selected').length > 1)
+            fail('duplicate result');
+        if (attempts.length > state.policy.maxAttemptsPerCell)
+            fail('retry budget');
+        if (attempts.some((a, i) => a.sequence !== i + 1))
+            fail('attempt ordering');
+    }
+    if (new Set(state.attempts.map((a) => a.id)).size !== state.attempts.length)
+        fail('attempt identity');
+    for (const assignment of state.assignments) {
+        const group = state.groups.find((g) => g.id === assignment.groupId);
+        if (!group ||
+            assignment.attemptIds.some((id) => {
+                const attempt = state.attempts.find((a) => a.id === id);
+                return (!attempt ||
+                    attempt.assignmentId !== assignment.id ||
+                    !group.cellIds.includes(attempt.cellId));
+            }))
+            fail('assignment membership');
+    }
+    for (const attempt of state.attempts.filter((a) => a.disposition === 'selected')) {
+        if (!attempt.report ||
+            attempt.report.verification !== 'valid' ||
+            attempt.report.cleanup !== 'confirmed' ||
+            attempt.report.environmentFailure !== null ||
+            !['completed', 'subject_failed'].includes(attempt.report.execution))
+            fail('selected evidence');
+    }
+}
+export function simulationOutput() {
+    return {
+        report: {
+            result: {
+                status: 'completed',
+                score: 1,
+                usage: {
+                    inputTokens: 1,
+                    outputTokens: 1,
+                    cacheReadTokens: 0,
+                    cacheWriteTokens: 0,
+                    reasoningTokens: 0,
+                    totalTokens: 2,
+                },
+                costUsd: 0.01,
+                durationMs: 1,
+                failureReason: null,
+                artifacts: [],
+            },
+            execution: 'completed',
+            verification: 'valid',
+            usage: 'complete',
+            cleanup: 'confirmed',
+            environmentFailure: null,
+        },
+        artifacts: [new TextEncoder().encode('mock subject artifact')],
+    };
+}
+class SimulatedLinkError extends Error {
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    const seed = Number(process.argv[2] ?? 1);
+    const result = await simulateFleet(seed);
+    process.stdout.write(`${JSON.stringify({ seed, elapsedTicks: result.elapsedTicks, ...result.summary, trace: result.trace }, null, 2)}\n`);
+}

package/node_modules/@maka/eval/dist/fleet-store.js

@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdir, open, readFile, rename, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { stableJsonStringify } from '@maka/core/canonical-json';
+import { createFleetState, transitionFleet, } from './fleet.js';
+export class FleetCoordinator {
+    persistence;
+    now;
+    static #owners = new WeakSet();
+    #tail = Promise.resolve();
+    #closed = false;
+    constructor(persistence, now) {
+        this.persistence = persistence;
+        this.now = now;
+    }
+    static async open(input) {
+        if (FleetCoordinator.#owners.has(input.persistence))
+            throw new Error('fleet persistence already has a coordinator');
+        FleetCoordinator.#owners.add(input.persistence);
+        try {
+            const initial = createFleetState(input.spec, input.policy);
+            const existing = await input.persistence.load();
+            if (existing &&
+                (existing.version !== initial.version ||
+                    stableJsonStringify(existing.spec) !== stableJsonStringify(initial.spec) ||
+                    stableJsonStringify(existing.policy) !== stableJsonStringify(initial.policy))) {
+                throw new Error('fleet run identity differs');
+            }
+            if (!existing)
+                await input.persistence.save(initial);
+            const coordinator = new FleetCoordinator(input.persistence, input.now);
+            await coordinator.command({ kind: 'recover' });
+            return coordinator;
+        }
+        catch (error) {
+            FleetCoordinator.#owners.delete(input.persistence);
+            throw error;
+        }
+    }
+    command(command) {
+        if (this.#closed)
+            return Promise.reject(new Error('coordinator closed'));
+        const captured = structuredClone(command);
+        const operation = this.#tail.then(async () => {
+            const state = await this.persistence.load();
+            if (!state)
+                throw new Error('fleet state missing');
+            if (captured.kind === 'report') {
+                for (const artifact of captured.report.artifacts)
+                    await this.persistence.getArtifact(artifact);
+            }
+            const next = transitionFleet(state, captured, this.now());
+            await this.persistence.save(next.state);
+            return next.reply;
+        });
+        this.#tail = operation.catch(() => undefined);
+        return operation;
+    }
+    async snapshot() {
+        await this.#tail;
+        const state = await this.persistence.load();
+        if (!state)
+            throw new Error('fleet state missing');
+        return state;
+    }
+    putArtifact(bytes) {
+        if (this.#closed)
+            return Promise.reject(new Error('coordinator closed'));
+        return this.persistence.putArtifact(bytes);
+    }
+    /** Stop accepting commands and drain commits before closing persistence or replacing this owner. */
+    async close() {
+        if (this.#closed) {
+            await this.#tail;
+            return;
+        }
+        this.#closed = true;
+        await this.#tail;
+        FleetCoordinator.#owners.delete(this.persistence);
+    }
+}
+export class MemoryFleetPersistence {
+    #state = null;
+    #artifacts = new Map();
+    async load() {
+        return structuredClone(this.#state);
+    }
+    async save(state) {
+        this.#state = structuredClone(state);
+    }
+    async putArtifact(bytes) {
+        const ref = reference(bytes);
+        this.#artifacts.set(ref.sha256, Uint8Array.from(bytes));
+        return ref;
+    }
+    async getArtifact(ref) {
+        validateReference(ref);
+        const bytes = this.#artifacts.get(ref.sha256);
+        if (!bytes)
+            throw new Error('artifact missing');
+        verifyArtifact(ref, bytes);
+        return Uint8Array.from(bytes);
+    }
+}
+/** Local coordinator storage. A stale writer lock requires explicit operator recovery. */
+export class FileFleetPersistence {
+    root;
+    release;
+    #tail = Promise.resolve();
+    #closing;
+    constructor(root, release) {
+        this.root = root;
+        this.release = release;
+    }
+    static async open(root) {
+        await mkdir(root, { recursive: true });
+        const lockPath = join(root, '.writer.lock');
+        const lock = await open(lockPath, 'wx', 0o600);
+        let closed = false;
+        return new FileFleetPersistence(root, async () => {
+            if (closed)
+                return;
+            closed = true;
+            await lock.close();
+            await unlink(lockPath);
+        });
+    }
+    async close() {
+        this.#closing ??= this.#tail.then(() => this.release());
+        await this.#closing;
+    }
+    #operation(operation) {
+        if (this.#closing)
+            return Promise.reject(new Error('fleet persistence closed'));
+        const running = this.#tail.then(operation);
+        this.#tail = running.catch(() => undefined);
+        return running;
+    }
+    load() {
+        return this.#operation(() => this.#load());
+    }
+    async #load() {
+        let raw;
+        try {
+            raw = await readFile(join(this.root, 'state.json'), 'utf8');
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return null;
+            throw error;
+        }
+        const envelope = JSON.parse(raw);
+        if (digest(Buffer.from(stableJsonStringify(envelope.state))) !== envelope.sha256) {
+            throw new Error('fleet state checksum mismatch');
+        }
+        if (envelope.state.version !== 'maka.eval.fleet.v1')
+            throw new Error('unsupported fleet state');
+        return envelope.state;
+    }
+    async save(state) {
+        const canonical = stableJsonStringify(state);
+        const bytes = Buffer.from(JSON.stringify({ sha256: digest(Buffer.from(canonical)), state }));
+        await this.#operation(() => atomicWrite(this.root, 'state.json', bytes));
+    }
+    async putArtifact(bytes) {
+        const captured = Uint8Array.from(bytes);
+        return this.#operation(async () => {
+            const ref = reference(captured);
+            const directory = join(this.root, 'artifacts');
+            await mkdir(directory, { recursive: true });
+            await atomicWrite(directory, ref.sha256, captured);
+            return ref;
+        });
+    }
+    async getArtifact(ref) {
+        validateReference(ref);
+        const captured = { ...ref };
+        return this.#operation(async () => {
+            const bytes = await readFile(join(this.root, 'artifacts', captured.sha256));
+            verifyArtifact(captured, bytes);
+            return bytes;
+        });
+    }
+}
+async function atomicWrite(directory, name, bytes) {
+    const temporary = join(directory, `.${name}.${randomUUID()}.tmp`);
+    const file = await open(temporary, 'wx', 0o600);
+    try {
+        await file.writeFile(bytes);
+        await file.sync();
+        await file.close();
+        await rename(temporary, join(directory, name));
+        if (process.platform !== 'win32') {
+            const parent = await open(directory, 'r');
+            try {
+                await parent.sync();
+            }
+            finally {
+                await parent.close();
+            }
+        }
+    }
+    finally {
+        await file.close().catch(() => undefined);
+        await unlink(temporary).catch((error) => {
+            if (error.code !== 'ENOENT')
+                throw error;
+        });
+    }
+}
+function digest(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
+}
+function reference(bytes) {
+    return { sha256: digest(bytes), bytes: bytes.byteLength };
+}
+function validateReference(ref) {
+    if (!/^[a-f0-9]{64}$/.test(ref.sha256) || !Number.isSafeInteger(ref.bytes) || ref.bytes < 0) {
+        throw new Error('invalid artifact reference');
+    }
+}
+function verifyArtifact(ref, bytes) {
+    if (bytes.byteLength !== ref.bytes || digest(bytes) !== ref.sha256)
+        throw new Error('artifact checksum mismatch');
+}

package/node_modules/@maka/eval/dist/fleet-worker.js

@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Transport-independent worker loop. Its outbox survives disconnection, not process death.
+ * The caller drives polling; no wall clock, timers, networking or cloud SDK is hidden here.
+ */
+export class FleetWorker {
+    description;
+    transport;
+    execute;
+    #runs = new Map();
+    #outbox = new Map();
+    #polling;
+    constructor(description, transport, execute) {
+        this.description = description;
+        this.transport = transport;
+        this.execute = execute;
+    }
+    poll() {
+        this.#polling ??= this.#poll().finally(() => {
+            this.#polling = undefined;
+        });
+        return this.#polling;
+    }
+    get bufferedReports() {
+        return this.#outbox.size;
+    }
+    get activeGroups() {
+        return this.#runs.size;
+    }
+    async #poll() {
+        await this.transport.command({ kind: 'register', worker: this.description });
+        const heartbeat = await this.transport.command({
+            kind: 'heartbeat',
+            workerId: this.description.id,
+            assignmentIds: [...this.#runs.keys()],
+        });
+        for (const work of heartbeat.work ?? [])
+            this.#adopt(work);
+        for (const [attemptId, output] of this.#outbox) {
+            const artifacts = [];
+            for (const bytes of output.artifacts)
+                artifacts.push(await this.transport.putArtifact(bytes));
+            await this.transport.command({
+                kind: 'report',
+                workerId: this.description.id,
+                attemptId,
+                report: { ...output.report, artifacts },
+            });
+            this.#outbox.delete(attemptId);
+        }
+        for (const [assignmentId, run] of this.#runs) {
+            if (!run.settled || run.work.cells.some((cell) => this.#outbox.has(cell.attemptId)))
+                continue;
+            await this.transport.command({ kind: 'finish', workerId: this.description.id, assignmentId });
+            this.#runs.delete(assignmentId);
+        }
+        while (this.#runs.size < this.description.groupSlots) {
+            const reply = await this.transport.command({ kind: 'claim', workerId: this.description.id });
+            if (!reply.assignment)
+                break;
+            for (const work of reply.work ?? [])
+                this.#adopt(work);
+        }
+    }
+    #adopt(work) {
+        if (this.#runs.has(work.assignment.id))
+            return;
+        const run = { work, settled: false, emitted: new Set() };
+        this.#runs.set(work.assignment.id, run);
+        const emit = (attemptId, output) => {
+            if (!work.cells.some((cell) => cell.attemptId === attemptId) || run.emitted.has(attemptId)) {
+                throw new Error('executor emitted an unknown or duplicate attempt');
+            }
+            run.emitted.add(attemptId);
+            this.#outbox.set(attemptId, structuredClone(output));
+        };
+        // A failed local operation is an execution uncertainty, not a provider classification.
+        void Promise.resolve()
+            .then(() => this.execute(work, emit))
+            .catch(() => undefined)
+            .finally(() => {
+            for (const cell of work.cells) {
+                if (run.emitted.has(cell.attemptId))
+                    continue;
+                emit(cell.attemptId, {
+                    report: {
+                        result: {
+                            status: 'infra_failed',
+                            score: null,
+                            usage: null,
+                            costUsd: null,
+                            durationMs: 0,
+                            failureReason: 'worker execution ended without a cell report',
+                            artifacts: [],
+                        },
+                        execution: 'unknown',
+                        verification: 'not_run',
+                        usage: 'missing',
+                        cleanup: 'unknown',
+                        environmentFailure: null,
+                    },
+                    artifacts: [],
+                });
+            }
+            run.settled = true;
+        });
+    }
+}