@mearl/client 2.20.6 → 2.21.0

package/dist/pluginTaskProtocol.d.ts

@@ -0,0 +1,47 @@
+import type { PluginExecutionError, PluginInvocationSource } from './generated-plugin-execution.js';
+import type { PluginOptions } from './plugin.js';
+export interface PluginTaskScope {
+    access: 'public' | 'toolbox' | 'runtime';
+    plugin?: string;
+    source: PluginInvocationSource;
+    bridge: 'service' | 'caller';
+    scopeId?: string;
+}
+export interface PluginTaskStartRequest {
+    plugin: string;
+    command: string;
+    kind?: 'command' | 'model';
+    input: Record<string, unknown>;
+    cwd: string;
+    environment: Record<string, string>;
+    options: PluginOptions;
+    executionTimeoutMs?: number;
+    idempotencyKey?: string;
+}
+export interface PluginTaskResumeRequest {
+    taskId: string;
+    environment: Record<string, string>;
+    revision?: 'original' | 'current';
+}
+export type PluginTaskWireMessage = {
+    type: 'request';
+    id: number;
+    method: string;
+    input: Record<string, unknown>;
+} | {
+    type: 'response';
+    id: number;
+    result?: unknown;
+    error?: PluginExecutionError;
+} | {
+    type: 'invoke';
+    id: number;
+    action: string;
+    payload: Record<string, unknown>;
+    options: PluginOptions;
+} | {
+    type: 'invoke-result';
+    id: number;
+    result?: unknown;
+    error?: PluginExecutionError;
+};

package/dist/pluginTaskProtocol.js

@@ -0,0 +1 @@
+export {};

package/dist/pluginTaskStore.d.ts

@@ -0,0 +1,53 @@
+import type { PluginTaskDiagnosticMetadata } from './generated-plugin-diagnostics.js';
+import type { PluginTask, PluginTaskEvent, PluginTaskEvents, PluginTaskResultChunk } from './generated-plugin-tasks.js';
+import type { PluginTaskCheckpoint } from './generated-plugin-recovery.js';
+import type { PluginOptions } from './plugin.js';
+export declare const PLUGIN_TASK_RESULT_CHUNK_BYTES: number;
+export interface PluginTaskRecord {
+    task: PluginTask;
+    events: PluginTaskEvent[];
+    dedupeKey?: string;
+    requestHash?: string;
+    access: 'public' | 'toolbox' | 'runtime';
+    checkpoint?: PluginTaskCheckpoint;
+    diagnostics?: PluginTaskDiagnosticMetadata;
+    execution?: {
+        entry: string;
+        options: PluginOptions;
+        bridge: 'service' | 'caller';
+        executionTimeoutMs?: number;
+    };
+    executorOwner?: {
+        pid: number;
+        processIdentity?: string;
+    };
+}
+export declare function pluginTaskExecutionUnconfirmed(record: PluginTaskRecord): boolean;
+export declare class PluginTaskStore {
+    readonly directory: string;
+    private activeTaskIds?;
+    private dedupeIndex?;
+    private readonly dedupeKeys;
+    constructor(directory?: string);
+    private forgetDedupe;
+    private indexDedupe;
+    /** The service owns writes; the index retains identities and reads the current matching record. */
+    findDedupe(key: string, activeOnly?: boolean): PluginTaskRecord | undefined;
+    private path;
+    read(taskId: string): PluginTaskRecord;
+    write(record: PluginTaskRecord): void;
+    private indexActive;
+    /** The service owns writes; only active identities remain indexed in memory. */
+    listActive(): PluginTaskRecord[];
+    update(taskId: string, update: Partial<PluginTask>, type?: PluginTaskEvent['type'], fields?: Pick<Partial<PluginTaskRecord>, 'checkpoint' | 'diagnostics'>): PluginTask;
+    list(): PluginTaskRecord[];
+    diagnosticRecords(taskId?: string): {
+        records: PluginTaskRecord[];
+        limited: boolean;
+    };
+    events(taskId: string, cursor: number): PluginTaskEvents;
+    saveResult(taskId: string, value: unknown): PluginTask['result'];
+    resultChunk(taskId: string, index: number): PluginTaskResultChunk;
+    recover(epoch: string): void;
+    prune(now?: number): void;
+}

package/dist/pluginTaskStore.js

@@ -0,0 +1,319 @@
+import { createHash } from 'node:crypto';
+import { closeSync, openSync, opendirSync, readFileSync, readSync, readdirSync, rmSync, statSync, } from 'node:fs';
+import { join } from 'node:path';
+import { isPluginTaskActive, isPluginTaskId } from './generated-plugin-tasks.js';
+import { writePluginHostFile } from './pluginHostFiles.js';
+import { pluginHostPaths } from './pluginHostPaths.js';
+import { isProcessGroupAlive, isProcessOwnerAlive } from '@mearl/daemon-core';
+export const PLUGIN_TASK_RESULT_CHUNK_BYTES = 192 * 1024;
+const MAX_EVENT_BYTES = 512 * 1024;
+const MAX_RESULT_BYTES = 64 * 1024 * 1024;
+const RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+export function pluginTaskExecutionUnconfirmed(record) {
+    const owner = record.executorOwner;
+    if (owner && (!Number.isSafeInteger(owner.pid) || owner.pid < 1))
+        return true;
+    if ((owner && isProcessOwnerAlive(owner)) ||
+        record.task.error?.details?.executionStopped === false ||
+        (process.platform === 'win32' && record.task.error?.code === 'PLUGIN_TASK_HOST_RESTARTED'))
+        return true;
+    if (owner && process.platform !== 'win32')
+        return isProcessGroupAlive(owner.pid);
+    return false;
+}
+export class PluginTaskStore {
+    directory;
+    activeTaskIds;
+    dedupeIndex;
+    dedupeKeys = new Map();
+    constructor(directory = pluginHostPaths().tasks) {
+        this.directory = directory;
+    }
+    forgetDedupe(taskId) {
+        const key = this.dedupeKeys.get(taskId);
+        if (!key)
+            return;
+        const ids = this.dedupeIndex?.get(key);
+        ids?.delete(taskId);
+        if (!ids?.size)
+            this.dedupeIndex?.delete(key);
+        this.dedupeKeys.delete(taskId);
+    }
+    indexDedupe(record) {
+        if (!this.dedupeIndex)
+            return;
+        const { dedupeKey: key, task } = record;
+        const keep = key && (!key.startsWith('active:') || isPluginTaskActive(task.state));
+        if (keep && this.dedupeKeys.get(task.taskId) === key)
+            return;
+        this.forgetDedupe(task.taskId);
+        if (!keep)
+            return;
+        const ids = this.dedupeIndex.get(key) ?? new Set();
+        ids.add(task.taskId);
+        this.dedupeIndex.set(key, ids);
+        this.dedupeKeys.set(task.taskId, key);
+    }
+    /** The service owns writes; the index retains identities and reads the current matching record. */
+    findDedupe(key, activeOnly = false) {
+        if (!this.dedupeIndex) {
+            this.dedupeIndex = new Map();
+            for (const record of this.list())
+                this.indexDedupe(record);
+        }
+        for (const taskId of this.dedupeIndex.get(key) ?? []) {
+            let record;
+            try {
+                record = this.read(taskId);
+            }
+            catch (error) {
+                if (error.code !== 'PLUGIN_TASK_NOT_FOUND')
+                    throw error;
+                this.forgetDedupe(taskId);
+                continue;
+            }
+            this.indexDedupe(record);
+            if (record.dedupeKey === key && (!activeOnly || isPluginTaskActive(record.task.state)))
+                return record;
+        }
+        return undefined;
+    }
+    path(taskId, file) {
+        if (!isPluginTaskId(taskId))
+            throw Object.assign(new Error('Invalid plugin task ID'), { code: 'PLUGIN_TASK_INVALID_ID' });
+        return join(this.directory, taskId, file);
+    }
+    read(taskId) {
+        let record;
+        try {
+            record = JSON.parse(readFileSync(this.path(taskId, 'task.json'), 'utf8'));
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                throw Object.assign(new Error('Plugin task was not found'), {
+                    code: 'PLUGIN_TASK_NOT_FOUND',
+                });
+            throw error;
+        }
+        if (record?.task?.taskId !== taskId || !Array.isArray(record.events))
+            throw new Error('Invalid persistent plugin task record');
+        return record;
+    }
+    write(record) {
+        writePluginHostFile(this.path(record.task.taskId, 'task.json'), JSON.stringify(record));
+        this.indexDedupe(record);
+        this.indexActive(record);
+    }
+    indexActive(record) {
+        if (isPluginTaskActive(record.task.state))
+            this.activeTaskIds?.add(record.task.taskId);
+        else
+            this.activeTaskIds?.delete(record.task.taskId);
+    }
+    /** The service owns writes; only active identities remain indexed in memory. */
+    listActive() {
+        if (!this.activeTaskIds) {
+            this.activeTaskIds = new Set();
+            for (const record of this.list())
+                this.indexActive(record);
+        }
+        const records = [];
+        for (const taskId of this.activeTaskIds) {
+            try {
+                const record = this.read(taskId);
+                this.indexActive(record);
+                if (isPluginTaskActive(record.task.state))
+                    records.push(record);
+            }
+            catch (error) {
+                if (error.code !== 'PLUGIN_TASK_NOT_FOUND')
+                    throw error;
+                this.activeTaskIds.delete(taskId);
+            }
+        }
+        return records;
+    }
+    update(taskId, update, type = 'state', fields = {}) {
+        const record = this.read(taskId);
+        Object.assign(record, fields);
+        record.task = {
+            ...record.task,
+            ...update,
+            sequence: record.task.sequence + 1,
+            updatedAt: Date.now(),
+        };
+        const event = {
+            sequence: record.task.sequence,
+            time: record.task.updatedAt,
+            type,
+            state: record.task.state,
+            ...(type === 'progress' ? { progress: record.task.progress } : {}),
+            ...(type === 'checkpoint' ? { checkpoint: record.task.checkpoint } : {}),
+        };
+        record.events.push(event);
+        let bytes = Buffer.byteLength(JSON.stringify(record.events));
+        while (record.events.length > 256 || (bytes > MAX_EVENT_BYTES && record.events.length > 1)) {
+            bytes -= Buffer.byteLength(JSON.stringify(record.events.shift())) + 1;
+        }
+        this.write(record);
+        return record.task;
+    }
+    list() {
+        let names;
+        try {
+            names = readdirSync(this.directory);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return [];
+            throw error;
+        }
+        return names.filter(isPluginTaskId).map(name => this.read(name));
+    }
+    diagnosticRecords(taskId) {
+        const records = [];
+        let bytes = 0, scanned = 0, limited = false;
+        if (taskId) {
+            let size;
+            try {
+                size = statSync(this.path(taskId, 'task.json')).size;
+            }
+            catch (error) {
+                if (error.code === 'ENOENT')
+                    return { records, limited };
+                throw error;
+            }
+            if (size > 16 * 1024 * 1024)
+                return { records, limited: true };
+            records.push(this.read(taskId));
+            bytes += size;
+        }
+        let directory;
+        try {
+            directory = opendirSync(this.directory);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return { records, limited };
+            throw error;
+        }
+        try {
+            let item;
+            while ((item = directory.readSync()) !== null) {
+                if (++scanned > 2048) {
+                    limited = true;
+                    break;
+                }
+                if (!isPluginTaskId(item.name) || item.name === taskId)
+                    continue;
+                try {
+                    const size = statSync(this.path(item.name, 'task.json')).size;
+                    if (records.length >= 512 || bytes + size > 16 * 1024 * 1024) {
+                        limited = true;
+                        break;
+                    }
+                    bytes += size;
+                    records.push(this.read(item.name));
+                }
+                catch {
+                    limited = true;
+                }
+            }
+        }
+        finally {
+            directory.closeSync();
+        }
+        return { records, limited };
+    }
+    events(taskId, cursor) {
+        const record = this.read(taskId);
+        if (!Number.isSafeInteger(cursor) || cursor < 0 || cursor > record.task.sequence)
+            throw new Error('Invalid plugin task event cursor');
+        const first = record.events[0]?.sequence ?? record.task.sequence + 1;
+        const events = [];
+        let bytes = 0;
+        for (const event of record.events) {
+            if (event.sequence <= cursor)
+                continue;
+            const size = Buffer.byteLength(JSON.stringify(event));
+            if (bytes + size > 192 * 1024 && events.length)
+                break;
+            events.push(event);
+            bytes += size;
+        }
+        return {
+            task: record.task,
+            events,
+            firstAvailableSequence: first,
+            nextCursor: events.at(-1)?.sequence ?? cursor,
+            gap: cursor < first - 1,
+        };
+    }
+    saveResult(taskId, value) {
+        const bytes = Buffer.from(JSON.stringify({ value }));
+        if (bytes.length > MAX_RESULT_BYTES)
+            throw Object.assign(new Error('Plugin task result exceeds 64 MiB'), {
+                code: 'PLUGIN_TASK_RESULT_TOO_LARGE',
+            });
+        writePluginHostFile(this.path(taskId, 'result.json'), bytes);
+        return {
+            bytes: bytes.length,
+            sha256: createHash('sha256').update(bytes).digest('hex'),
+            chunks: Math.ceil(bytes.length / PLUGIN_TASK_RESULT_CHUNK_BYTES),
+        };
+    }
+    resultChunk(taskId, index) {
+        const task = this.read(taskId).task;
+        if (task.state !== 'success' || !task.result)
+            throw new Error('Plugin task result is not available');
+        if (!Number.isSafeInteger(index) || index < 0 || index >= task.result.chunks)
+            throw new Error('Invalid plugin task result chunk');
+        const file = this.path(taskId, 'result.json');
+        if (statSync(file).size !== task.result.bytes)
+            throw new Error('Plugin task result size does not match its record');
+        const offset = index * PLUGIN_TASK_RESULT_CHUNK_BYTES;
+        const buffer = Buffer.alloc(Math.min(PLUGIN_TASK_RESULT_CHUNK_BYTES, task.result.bytes - offset));
+        const descriptor = openSync(file, 'r');
+        try {
+            if (readSync(descriptor, buffer, 0, buffer.length, offset) !== buffer.length)
+                throw new Error('Plugin task result is incomplete');
+        }
+        finally {
+            closeSync(descriptor);
+        }
+        return { taskId, index, data: buffer.toString('base64'), ...task.result };
+    }
+    recover(epoch) {
+        this.activeTaskIds = new Set();
+        for (const record of this.list()) {
+            this.indexActive(record);
+            const { task } = record;
+            if (isPluginTaskActive(task.state) && task.host.epoch !== epoch) {
+                this.update(task.taskId, {
+                    state: 'interrupted',
+                    error: {
+                        code: 'PLUGIN_TASK_HOST_RESTARTED',
+                        message: 'The plugin execution service restarted; external operations may need reconciliation',
+                    },
+                });
+            }
+            else if (!isPluginTaskActive(task.state) &&
+                Date.now() - task.updatedAt > RETENTION_MS &&
+                !pluginTaskExecutionUnconfirmed(record)) {
+                rmSync(join(this.directory, task.taskId), { recursive: true, force: true });
+                this.forgetDedupe(task.taskId);
+            }
+        }
+    }
+    prune(now = Date.now()) {
+        for (const record of this.list()) {
+            const { task } = record;
+            if (!isPluginTaskActive(task.state) &&
+                now - task.updatedAt > RETENTION_MS &&
+                !pluginTaskExecutionUnconfirmed(record)) {
+                rmSync(join(this.directory, task.taskId), { recursive: true, force: true });
+                this.forgetDedupe(task.taskId);
+            }
+        }
+    }
+}

package/dist/pluginTaskWire.d.ts

@@ -0,0 +1,4 @@
+import type { Socket } from 'node:net';
+import type { PluginTaskWireMessage } from './pluginTaskProtocol.js';
+export declare function readPluginTaskMessages(socket: Socket, receive: (message: PluginTaskWireMessage) => void): void;
+export declare function writePluginTaskMessage(socket: Socket, message: PluginTaskWireMessage): Promise<void>;

package/dist/pluginTaskWire.js

@@ -0,0 +1,42 @@
+import { StringDecoder } from 'node:string_decoder';
+const MAX_MESSAGE_BYTES = 96 * 1024 * 1024;
+export function readPluginTaskMessages(socket, receive) {
+    const decoder = new StringDecoder('utf8');
+    let buffer = '';
+    socket.on('data', chunk => {
+        try {
+            buffer += decoder.write(chunk);
+            let end;
+            while ((end = buffer.indexOf('\n')) >= 0) {
+                const line = buffer.slice(0, end);
+                buffer = buffer.slice(end + 1);
+                if (Buffer.byteLength(line) > MAX_MESSAGE_BYTES)
+                    throw new Error('Plugin host message is too large');
+                const message = JSON.parse(line);
+                if (!message || typeof message !== 'object' || !Number.isSafeInteger(message.id))
+                    throw new Error('Invalid plugin host message');
+                receive(message);
+            }
+            if (Buffer.byteLength(buffer) > MAX_MESSAGE_BYTES)
+                throw new Error('Plugin host message buffer exceeded its limit');
+        }
+        catch (error) {
+            socket.destroy(error instanceof Error ? error : new Error(String(error)));
+        }
+    });
+}
+export function writePluginTaskMessage(socket, message) {
+    return new Promise((resolve, reject) => {
+        try {
+            const text = JSON.stringify(message) + '\n';
+            if (Buffer.byteLength(text) > MAX_MESSAGE_BYTES)
+                throw new Error('Plugin host message is too large');
+            if (socket.destroyed)
+                throw new Error('Plugin host connection is closed');
+            socket.write(text, error => (error ? reject(error) : resolve()));
+        }
+        catch (error) {
+            reject(error);
+        }
+    });
+}

package/dist/pluginTasks.d.ts

@@ -0,0 +1,51 @@
+import { PluginHostConnection } from './pluginHostConnection.js';
+import type { PluginHostClientOptions, HostStatus as PluginHostStatus } from './pluginHostConnection.js';
+import type { PluginTaskCheckpoint } from './generated-plugin-recovery.js';
+import type { PluginDiagnosticOptions, PluginDiagnosticReport } from './generated-plugin-diagnostics.js';
+import type { PluginTask, PluginTaskEvents, PluginTaskListResult, PluginTaskResultChunk, PluginTaskStartResult } from './generated-plugin-tasks.js';
+import type { PluginInvocation, PluginOptions } from './plugin.js';
+export type { PluginTask, PluginTaskEvents, PluginTaskListResult, PluginTaskStartResult };
+export interface PluginTaskStartOptions extends PluginOptions {
+    cwd?: string;
+    environment?: NodeJS.ProcessEnv;
+    executionTimeoutMs?: number;
+    idempotencyKey?: string;
+}
+export interface PluginTaskWaitOptions {
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    readResult?: boolean;
+    onProgress?: (progress: PluginTask['progress'], task: PluginTask) => void;
+}
+export type PluginTaskClientOptions = PluginHostClientOptions;
+/** Background execution stays on this host; disconnecting a caller never cancels its tasks. */
+export declare class PluginTaskClient {
+    private readonly configuration;
+    readonly connection: PluginHostConnection;
+    constructor(configuration?: PluginTaskClientOptions);
+    private request;
+    status(): Promise<PluginHostStatus>;
+    diagnose(options?: PluginDiagnosticOptions): Promise<PluginDiagnosticReport>;
+    checkpoint(taskId: string): Promise<PluginTaskCheckpoint | null>;
+    resume(taskId: string, options?: {
+        environment?: NodeJS.ProcessEnv;
+        revision?: 'original' | 'current';
+    }): Promise<PluginTaskStartResult>;
+    start(plugin: string, invocation: PluginInvocation, options?: PluginTaskStartOptions): Promise<PluginTaskStartResult>;
+    get(taskId: string): Promise<PluginTask>;
+    list(activeOnly?: boolean, page?: {
+        cursor?: string;
+        limit?: number;
+    }): Promise<PluginTaskListResult>;
+    cancel(taskId: string): Promise<PluginTask>;
+    events(taskId: string, cursor?: number, waitMs?: number): Promise<PluginTaskEvents>;
+    resultChunk(taskId: string, index: number): Promise<PluginTaskResultChunk>;
+    result(taskId: string): Promise<unknown>;
+    subscribe(taskId: string, options?: {
+        cursor?: number;
+        signal?: AbortSignal;
+    }): AsyncGenerator<PluginTaskEvents>;
+    wait(taskId: string, options?: PluginTaskWaitOptions): Promise<unknown>;
+    stopHost(): Promise<void>;
+    disconnect(): void;
+}