@mearl/client 2.20.6 → 2.22.0

package/dist/plugins/runtime/pluginTaskStore.d.ts

@@ -0,0 +1,53 @@
+import type { PluginTaskDiagnosticMetadata } from '../../generated/plugin-diagnostics.js';
+import type { PluginTask, PluginTaskEvent, PluginTaskEvents, PluginTaskResultChunk } from '../../generated/plugin-tasks.js';
+import type { PluginTaskCheckpoint } from '../../generated/plugin-recovery.js';
+import type { PluginOptions } from '../sdk/plugin.js';
+export declare const PLUGIN_TASK_RESULT_CHUNK_BYTES: number;
+export interface PluginTaskRecord {
+    task: PluginTask;
+    events: PluginTaskEvent[];
+    dedupeKey?: string;
+    requestHash?: string;
+    access: 'public' | 'toolbox' | 'runtime';
+    checkpoint?: PluginTaskCheckpoint;
+    diagnostics?: PluginTaskDiagnosticMetadata;
+    execution?: {
+        entry: string;
+        options: PluginOptions;
+        bridge: 'service' | 'caller';
+        executionTimeoutMs?: number;
+    };
+    executorOwner?: {
+        pid: number;
+        processIdentity?: string;
+    };
+}
+export declare function pluginTaskExecutionUnconfirmed(record: PluginTaskRecord): boolean;
+export declare class PluginTaskStore {
+    readonly directory: string;
+    private activeTaskIds?;
+    private dedupeIndex?;
+    private readonly dedupeKeys;
+    constructor(directory?: string);
+    private forgetDedupe;
+    private indexDedupe;
+    /** The service owns writes; the index retains identities and reads the current matching record. */
+    findDedupe(key: string, activeOnly?: boolean): PluginTaskRecord | undefined;
+    private path;
+    read(taskId: string): PluginTaskRecord;
+    write(record: PluginTaskRecord): void;
+    private indexActive;
+    /** The service owns writes; only active identities remain indexed in memory. */
+    listActive(): PluginTaskRecord[];
+    update(taskId: string, update: Partial<PluginTask>, type?: PluginTaskEvent['type'], fields?: Pick<Partial<PluginTaskRecord>, 'checkpoint' | 'diagnostics'>): PluginTask;
+    list(): PluginTaskRecord[];
+    diagnosticRecords(taskId?: string): {
+        records: PluginTaskRecord[];
+        limited: boolean;
+    };
+    events(taskId: string, cursor: number): PluginTaskEvents;
+    saveResult(taskId: string, value: unknown): PluginTask['result'];
+    resultChunk(taskId: string, index: number): PluginTaskResultChunk;
+    recover(epoch: string): void;
+    prune(now?: number): void;
+}

package/dist/plugins/runtime/pluginTaskStore.js

@@ -0,0 +1,319 @@
+import { createHash } from 'node:crypto';
+import { closeSync, openSync, opendirSync, readFileSync, readSync, readdirSync, rmSync, statSync, } from 'node:fs';
+import { join } from 'node:path';
+import { isPluginTaskActive, isPluginTaskId } from '../../generated/plugin-tasks.js';
+import { writePluginHostFile } from './pluginHostFiles.js';
+import { pluginHostPaths } from './pluginHostPaths.js';
+import { isProcessGroupAlive, isProcessOwnerAlive } from '@mearl/daemon-core';
+export const PLUGIN_TASK_RESULT_CHUNK_BYTES = 192 * 1024;
+const MAX_EVENT_BYTES = 512 * 1024;
+const MAX_RESULT_BYTES = 64 * 1024 * 1024;
+const RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+export function pluginTaskExecutionUnconfirmed(record) {
+    const owner = record.executorOwner;
+    if (owner && (!Number.isSafeInteger(owner.pid) || owner.pid < 1))
+        return true;
+    if ((owner && isProcessOwnerAlive(owner)) ||
+        record.task.error?.details?.executionStopped === false ||
+        (process.platform === 'win32' && record.task.error?.code === 'PLUGIN_TASK_HOST_RESTARTED'))
+        return true;
+    if (owner && process.platform !== 'win32')
+        return isProcessGroupAlive(owner.pid);
+    return false;
+}
+export class PluginTaskStore {
+    directory;
+    activeTaskIds;
+    dedupeIndex;
+    dedupeKeys = new Map();
+    constructor(directory = pluginHostPaths().tasks) {
+        this.directory = directory;
+    }
+    forgetDedupe(taskId) {
+        const key = this.dedupeKeys.get(taskId);
+        if (!key)
+            return;
+        const ids = this.dedupeIndex?.get(key);
+        ids?.delete(taskId);
+        if (!ids?.size)
+            this.dedupeIndex?.delete(key);
+        this.dedupeKeys.delete(taskId);
+    }
+    indexDedupe(record) {
+        if (!this.dedupeIndex)
+            return;
+        const { dedupeKey: key, task } = record;
+        const keep = key && (!key.startsWith('active:') || isPluginTaskActive(task.state));
+        if (keep && this.dedupeKeys.get(task.taskId) === key)
+            return;
+        this.forgetDedupe(task.taskId);
+        if (!keep)
+            return;
+        const ids = this.dedupeIndex.get(key) ?? new Set();
+        ids.add(task.taskId);
+        this.dedupeIndex.set(key, ids);
+        this.dedupeKeys.set(task.taskId, key);
+    }
+    /** The service owns writes; the index retains identities and reads the current matching record. */
+    findDedupe(key, activeOnly = false) {
+        if (!this.dedupeIndex) {
+            this.dedupeIndex = new Map();
+            for (const record of this.list())
+                this.indexDedupe(record);
+        }
+        for (const taskId of this.dedupeIndex.get(key) ?? []) {
+            let record;
+            try {
+                record = this.read(taskId);
+            }
+            catch (error) {
+                if (error.code !== 'PLUGIN_TASK_NOT_FOUND')
+                    throw error;
+                this.forgetDedupe(taskId);
+                continue;
+            }
+            this.indexDedupe(record);
+            if (record.dedupeKey === key && (!activeOnly || isPluginTaskActive(record.task.state)))
+                return record;
+        }
+        return undefined;
+    }
+    path(taskId, file) {
+        if (!isPluginTaskId(taskId))
+            throw Object.assign(new Error('Invalid plugin task ID'), { code: 'PLUGIN_TASK_INVALID_ID' });
+        return join(this.directory, taskId, file);
+    }
+    read(taskId) {
+        let record;
+        try {
+            record = JSON.parse(readFileSync(this.path(taskId, 'task.json'), 'utf8'));
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                throw Object.assign(new Error('Plugin task was not found'), {
+                    code: 'PLUGIN_TASK_NOT_FOUND',
+                });
+            throw error;
+        }
+        if (record?.task?.taskId !== taskId || !Array.isArray(record.events))
+            throw new Error('Invalid persistent plugin task record');
+        return record;
+    }
+    write(record) {
+        writePluginHostFile(this.path(record.task.taskId, 'task.json'), JSON.stringify(record));
+        this.indexDedupe(record);
+        this.indexActive(record);
+    }
+    indexActive(record) {
+        if (isPluginTaskActive(record.task.state))
+            this.activeTaskIds?.add(record.task.taskId);
+        else
+            this.activeTaskIds?.delete(record.task.taskId);
+    }
+    /** The service owns writes; only active identities remain indexed in memory. */
+    listActive() {
+        if (!this.activeTaskIds) {
+            this.activeTaskIds = new Set();
+            for (const record of this.list())
+                this.indexActive(record);
+        }
+        const records = [];
+        for (const taskId of this.activeTaskIds) {
+            try {
+                const record = this.read(taskId);
+                this.indexActive(record);
+                if (isPluginTaskActive(record.task.state))
+                    records.push(record);
+            }
+            catch (error) {
+                if (error.code !== 'PLUGIN_TASK_NOT_FOUND')
+                    throw error;
+                this.activeTaskIds.delete(taskId);
+            }
+        }
+        return records;
+    }
+    update(taskId, update, type = 'state', fields = {}) {
+        const record = this.read(taskId);
+        Object.assign(record, fields);
+        record.task = {
+            ...record.task,
+            ...update,
+            sequence: record.task.sequence + 1,
+            updatedAt: Date.now(),
+        };
+        const event = {
+            sequence: record.task.sequence,
+            time: record.task.updatedAt,
+            type,
+            state: record.task.state,
+            ...(type === 'progress' ? { progress: record.task.progress } : {}),
+            ...(type === 'checkpoint' ? { checkpoint: record.task.checkpoint } : {}),
+        };
+        record.events.push(event);
+        let bytes = Buffer.byteLength(JSON.stringify(record.events));
+        while (record.events.length > 256 || (bytes > MAX_EVENT_BYTES && record.events.length > 1)) {
+            bytes -= Buffer.byteLength(JSON.stringify(record.events.shift())) + 1;
+        }
+        this.write(record);
+        return record.task;
+    }
+    list() {
+        let names;
+        try {
+            names = readdirSync(this.directory);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return [];
+            throw error;
+        }
+        return names.filter(isPluginTaskId).map(name => this.read(name));
+    }
+    diagnosticRecords(taskId) {
+        const records = [];
+        let bytes = 0, scanned = 0, limited = false;
+        if (taskId) {
+            let size;
+            try {
+                size = statSync(this.path(taskId, 'task.json')).size;
+            }
+            catch (error) {
+                if (error.code === 'ENOENT')
+                    return { records, limited };
+                throw error;
+            }
+            if (size > 16 * 1024 * 1024)
+                return { records, limited: true };
+            records.push(this.read(taskId));
+            bytes += size;
+        }
+        let directory;
+        try {
+            directory = opendirSync(this.directory);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT')
+                return { records, limited };
+            throw error;
+        }
+        try {
+            let item;
+            while ((item = directory.readSync()) !== null) {
+                if (++scanned > 2048) {
+                    limited = true;
+                    break;
+                }
+                if (!isPluginTaskId(item.name) || item.name === taskId)
+                    continue;
+                try {
+                    const size = statSync(this.path(item.name, 'task.json')).size;
+                    if (records.length >= 512 || bytes + size > 16 * 1024 * 1024) {
+                        limited = true;
+                        break;
+                    }
+                    bytes += size;
+                    records.push(this.read(item.name));
+                }
+                catch {
+                    limited = true;
+                }
+            }
+        }
+        finally {
+            directory.closeSync();
+        }
+        return { records, limited };
+    }
+    events(taskId, cursor) {
+        const record = this.read(taskId);
+        if (!Number.isSafeInteger(cursor) || cursor < 0 || cursor > record.task.sequence)
+            throw new Error('Invalid plugin task event cursor');
+        const first = record.events[0]?.sequence ?? record.task.sequence + 1;
+        const events = [];
+        let bytes = 0;
+        for (const event of record.events) {
+            if (event.sequence <= cursor)
+                continue;
+            const size = Buffer.byteLength(JSON.stringify(event));
+            if (bytes + size > 192 * 1024 && events.length)
+                break;
+            events.push(event);
+            bytes += size;
+        }
+        return {
+            task: record.task,
+            events,
+            firstAvailableSequence: first,
+            nextCursor: events.at(-1)?.sequence ?? cursor,
+            gap: cursor < first - 1,
+        };
+    }
+    saveResult(taskId, value) {
+        const bytes = Buffer.from(JSON.stringify({ value }));
+        if (bytes.length > MAX_RESULT_BYTES)
+            throw Object.assign(new Error('Plugin task result exceeds 64 MiB'), {
+                code: 'PLUGIN_TASK_RESULT_TOO_LARGE',
+            });
+        writePluginHostFile(this.path(taskId, 'result.json'), bytes);
+        return {
+            bytes: bytes.length,
+            sha256: createHash('sha256').update(bytes).digest('hex'),
+            chunks: Math.ceil(bytes.length / PLUGIN_TASK_RESULT_CHUNK_BYTES),
+        };
+    }
+    resultChunk(taskId, index) {
+        const task = this.read(taskId).task;
+        if (task.state !== 'success' || !task.result)
+            throw new Error('Plugin task result is not available');
+        if (!Number.isSafeInteger(index) || index < 0 || index >= task.result.chunks)
+            throw new Error('Invalid plugin task result chunk');
+        const file = this.path(taskId, 'result.json');
+        if (statSync(file).size !== task.result.bytes)
+            throw new Error('Plugin task result size does not match its record');
+        const offset = index * PLUGIN_TASK_RESULT_CHUNK_BYTES;
+        const buffer = Buffer.alloc(Math.min(PLUGIN_TASK_RESULT_CHUNK_BYTES, task.result.bytes - offset));
+        const descriptor = openSync(file, 'r');
+        try {
+            if (readSync(descriptor, buffer, 0, buffer.length, offset) !== buffer.length)
+                throw new Error('Plugin task result is incomplete');
+        }
+        finally {
+            closeSync(descriptor);
+        }
+        return { taskId, index, data: buffer.toString('base64'), ...task.result };
+    }
+    recover(epoch) {
+        this.activeTaskIds = new Set();
+        for (const record of this.list()) {
+            this.indexActive(record);
+            const { task } = record;
+            if (isPluginTaskActive(task.state) && task.host.epoch !== epoch) {
+                this.update(task.taskId, {
+                    state: 'interrupted',
+                    error: {
+                        code: 'PLUGIN_TASK_HOST_RESTARTED',
+                        message: 'The plugin execution service restarted; external operations may need reconciliation',
+                    },
+                });
+            }
+            else if (!isPluginTaskActive(task.state) &&
+                Date.now() - task.updatedAt > RETENTION_MS &&
+                !pluginTaskExecutionUnconfirmed(record)) {
+                rmSync(join(this.directory, task.taskId), { recursive: true, force: true });
+                this.forgetDedupe(task.taskId);
+            }
+        }
+    }
+    prune(now = Date.now()) {
+        for (const record of this.list()) {
+            const { task } = record;
+            if (!isPluginTaskActive(task.state) &&
+                now - task.updatedAt > RETENTION_MS &&
+                !pluginTaskExecutionUnconfirmed(record)) {
+                rmSync(join(this.directory, task.taskId), { recursive: true, force: true });
+                this.forgetDedupe(task.taskId);
+            }
+        }
+    }
+}

package/dist/plugins/runtime/pythonPlugin.d.ts

@@ -0,0 +1,31 @@
+import type { PluginOptions, PluginApp } from '../sdk/plugin.js';
+import type { PluginCommandContext } from './pluginExecution.js';
+export interface PythonPluginCommand {
+    description: string;
+    signature: string;
+    toolboxOnly: boolean;
+    runtimeOnly: boolean;
+    recoveryOnly: boolean;
+    migrationOnly: boolean;
+}
+export interface PythonPluginDescriptor {
+    commands: PythonPluginCommand[];
+    description?: string;
+    name?: string;
+    toolbox?: unknown;
+    pageRuntime?: unknown;
+    appProfiles?: unknown;
+    stateStorage?: unknown;
+    taskRecovery?: unknown;
+}
+export interface PythonPluginApp {
+    state?: PluginApp['state'];
+    model?: PluginApp['model'];
+    checkpoint?: PluginApp['checkpoint'];
+    page?: PluginApp['page'];
+    readonly options: Readonly<PluginOptions>;
+    invoke(action: string, payload?: Record<string, any>, options?: PluginOptions): Promise<any>;
+    progress(message: string): void;
+}
+export declare function describePythonPlugin(entry: string, options: Readonly<PluginOptions>): Promise<PythonPluginDescriptor>;
+export declare function runPythonPluginCommand(entry: string, command: string, input: Record<string, any> | import('../../generated/plugin-recovery.js').PluginCheckpointMigrationInput, app: PythonPluginApp, context: PluginCommandContext, recovery?: boolean | 'migration', expectedDescriptor?: PythonPluginDescriptor, runtimeOnly?: boolean): Promise<unknown>;