@myagentroam/node 0.1.3 → 0.1.6

package/dist/service/workspace-content-search-service.js

@@ -0,0 +1,477 @@
+import { spawn } from 'node:child_process';
+import { createReadStream } from 'node:fs';
+import { lstat, realpath } from 'node:fs/promises';
+import { isAbsolute, join, relative, sep } from 'node:path';
+const RESULT_LIMIT = 200;
+const PER_FILE_LIMIT = 20;
+const FILE_BYTE_LIMIT = 2 * 1024 * 1024;
+const SCAN_BYTE_LIMIT = 128 * 1024 * 1024;
+const OUTPUT_BYTE_LIMIT = 2 * 1024 * 1024;
+const TIMEOUT_MS = 5_000;
+const WINDOWS_ARGUMENT_BATCH_BYTES = 24 * 1024;
+const POSIX_ARGUMENT_BATCH_BYTES = 96 * 1024;
+const SEARCH_CONCURRENCY = 2;
+const SEARCH_QUEUE_LIMIT = 16;
+const CANDIDATE_VALIDATION_CONCURRENCY = 32;
+let activeSearches = 0;
+const searchWaiters = [];
+async function acquireSearch(deadline) {
+    if (activeSearches >= SEARCH_CONCURRENCY) {
+        if (searchWaiters.length >= SEARCH_QUEUE_LIMIT)
+            throw new Error('FILE_CONTENT_SEARCH_BUSY');
+        await new Promise((resolve, reject) => {
+            const resume = () => {
+                clearTimeout(timer);
+                resolve();
+            };
+            const timer = setTimeout(() => {
+                const index = searchWaiters.indexOf(resume);
+                if (index >= 0)
+                    searchWaiters.splice(index, 1);
+                reject(new Error('CONTENT_SEARCH_TIMEOUT'));
+            }, Math.max(1, deadline - Date.now()));
+            searchWaiters.push(resume);
+        });
+    }
+    activeSearches += 1;
+    return () => {
+        activeSearches -= 1;
+        searchWaiters.shift()?.();
+    };
+}
+export class WorkspaceContentSearchService {
+    files;
+    platform;
+    pending = new Map();
+    constructor(files, platform = process.platform) {
+        this.files = files;
+        this.platform = platform;
+    }
+    search(workspace, input) {
+        if (typeof input.query !== 'string' || input.query.length < 1 || input.query.length > 256)
+            throw new Error('FILE_CONTENT_SEARCH_INVALID');
+        const key = `${workspace.id}\0${input.query.toLocaleLowerCase()}`;
+        const existing = this.pending.get(key);
+        if (existing !== undefined)
+            return existing;
+        const task = this.perform(workspace, input.query).finally(() => this.pending.delete(key));
+        this.pending.set(key, task);
+        return task;
+    }
+    async perform(workspace, query) {
+        const deadline = Date.now() + TIMEOUT_MS;
+        const release = await acquireSearch(deadline);
+        try {
+            const prepared = await this.candidates(workspace, deadline);
+            if (prepared.files.length === 0)
+                return {
+                    matches: [],
+                    engine: 'NODE',
+                    truncated: prepared.truncated || Date.now() >= deadline,
+                    skippedFiles: prepared.skippedFiles
+                };
+            for (const engine of this.engines()) {
+                try {
+                    const matches = await this.nativeSearch(engine, workspace.path, prepared.files, query, deadline);
+                    return {
+                        matches: matches.slice(0, RESULT_LIMIT),
+                        engine,
+                        truncated: prepared.truncated || matches.length > RESULT_LIMIT || Date.now() >= deadline,
+                        skippedFiles: prepared.skippedFiles
+                    };
+                }
+                catch {
+                    // Missing, incompatible or failed native tools safely fall through.
+                }
+            }
+            const fallback = await nodeSearch(prepared.files, query, deadline);
+            return {
+                matches: fallback.matches.slice(0, RESULT_LIMIT),
+                engine: 'NODE',
+                truncated: prepared.truncated || fallback.matches.length > RESULT_LIMIT || Date.now() >= deadline,
+                skippedFiles: prepared.skippedFiles + fallback.skippedFiles
+            };
+        }
+        finally {
+            release();
+        }
+    }
+    engines() {
+        return this.platform === 'win32' ? ['RIPGREP', 'POWERSHELL'] : ['RIPGREP', 'GREP'];
+    }
+    async candidates(workspace, deadline) {
+        const root = await realpath(workspace.path);
+        let candidates;
+        try {
+            candidates = await beforeDeadline(this.files.index(workspace).contentCandidates(), deadline);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message === 'CONTENT_SEARCH_TIMEOUT')
+                return { files: [], skippedFiles: 0, truncated: true };
+            throw error;
+        }
+        const files = [];
+        let skippedFiles = 0;
+        let bytes = 0;
+        let truncated = candidates.truncated;
+        for (let offset = 0; offset < candidates.entries.length; offset += CANDIDATE_VALIDATION_CONCURRENCY) {
+            if (Date.now() >= deadline) {
+                truncated = true;
+                break;
+            }
+            const validated = await Promise.all(candidates.entries
+                .slice(offset, offset + CANDIDATE_VALIDATION_CONCURRENCY)
+                .map((entry) => validateCandidate(root, entry.path)));
+            for (const candidate of validated) {
+                if (candidate === undefined) {
+                    skippedFiles += 1;
+                    continue;
+                }
+                if (bytes + candidate.size > SCAN_BYTE_LIMIT) {
+                    truncated = true;
+                    break;
+                }
+                bytes += candidate.size;
+                files.push({ path: candidate.path, absolutePath: candidate.absolutePath });
+            }
+            if (truncated)
+                break;
+        }
+        return { files, skippedFiles, truncated };
+    }
+    async nativeSearch(engine, root, files, query, deadline) {
+        const matches = [];
+        const argumentBatchBytes = this.platform === 'win32' ? WINDOWS_ARGUMENT_BATCH_BYTES : POSIX_ARGUMENT_BATCH_BYTES;
+        for (const batch of batches(files, argumentBatchBytes)) {
+            if (Date.now() >= deadline || matches.length > RESULT_LIMIT)
+                break;
+            const command = commandFor(engine, batch, query);
+            const result = await runCommand(command.file, command.args, root, deadline, engine === 'RIPGREP');
+            if (!command.validCodes.includes(result.code))
+                throw new Error('CONTENT_SEARCH_COMMAND_FAILED');
+            const paths = new Map(batch.flatMap((file) => [
+                [normalizeResultPath(file.path, this.platform), file.path],
+                [normalizeResultPath(file.absolutePath, this.platform), file.path]
+            ]));
+            matches.push(...parseOutput(engine, result.stdout, query).map((match) => ({
+                ...match,
+                path: paths.get(normalizeResultPath(match.path, this.platform)) ??
+                    (() => {
+                        throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+                    })()
+            })));
+        }
+        return limitPerFile(matches);
+    }
+}
+async function validateCandidate(root, path) {
+    try {
+        const absolutePath = await realpath(join(root, path));
+        const inside = relative(root, absolutePath);
+        if (inside === '..' || inside.startsWith(`..${sep}`) || isAbsolute(inside))
+            return undefined;
+        const metadata = await lstat(absolutePath);
+        return metadata.isFile() && metadata.size <= FILE_BYTE_LIMIT
+            ? { path, absolutePath, size: metadata.size }
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function beforeDeadline(promise, deadline) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error('CONTENT_SEARCH_TIMEOUT')), Math.max(1, deadline - Date.now()));
+        promise.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+function normalizeResultPath(path, platform) {
+    const normalized = path.replaceAll('\\', '/');
+    return platform === 'win32' ? normalized.toLocaleLowerCase() : normalized;
+}
+function batches(files, maximumBytes) {
+    const result = [];
+    let current = [];
+    let bytes = 0;
+    for (const file of files) {
+        const size = Buffer.byteLength(file.path) + 1;
+        if (current.length > 0 && bytes + size > maximumBytes) {
+            result.push(current);
+            current = [];
+            bytes = 0;
+        }
+        current.push(file);
+        bytes += size;
+    }
+    if (current.length > 0)
+        result.push(current);
+    return result;
+}
+function commandFor(engine, files, query) {
+    const paths = files.map((file) => file.path);
+    if (engine === 'RIPGREP')
+        return {
+            file: 'rg',
+            args: [
+                '--json',
+                '--no-config',
+                '--fixed-strings',
+                '--ignore-case',
+                '--max-count',
+                String(PER_FILE_LIMIT),
+                '--',
+                query,
+                ...paths
+            ],
+            validCodes: [0, 1]
+        };
+    if (engine === 'GREP')
+        return {
+            file: 'grep',
+            args: [
+                '-I',
+                '-i',
+                '-n',
+                '-H',
+                '-F',
+                '-Z',
+                '-m',
+                String(PER_FILE_LIMIT),
+                '-e',
+                query,
+                '--',
+                ...paths
+            ],
+            validCodes: [0, 1]
+        };
+    const script = [
+        '[Console]::OutputEncoding=[System.Text.Encoding]::UTF8;',
+        '$q=$args[0];',
+        '$args | Select-Object -Skip 1 | ForEach-Object {',
+        'Select-String -LiteralPath $_ -SimpleMatch -CaseSensitive:$false -Pattern $q | Select-Object -First 20 | ForEach-Object {',
+        '[Console]::Out.WriteLine((@{path=$_.Path;line=$_.LineNumber;column=($_.Line.ToLowerInvariant().IndexOf($q.ToLowerInvariant())+1);preview=$_.Line}|ConvertTo-Json -Compress))',
+        '}}'
+    ].join('');
+    return {
+        file: 'powershell.exe',
+        args: ['-NoProfile', '-NonInteractive', '-Command', script, query, ...paths],
+        validCodes: [0]
+    };
+}
+async function runCommand(file, args, cwd, deadline, stopAfterResultLimit = false) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(file, args, {
+            cwd,
+            windowsHide: true,
+            stdio: ['ignore', 'pipe', 'ignore']
+        });
+        const chunks = [];
+        let pending = Buffer.alloc(0);
+        let bytes = 0;
+        let matches = 0;
+        let stopped = false;
+        const timer = setTimeout(() => child.kill(), Math.max(1, deadline - Date.now()));
+        child.once('error', (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+        child.stdout.on('data', (chunk) => {
+            bytes += chunk.length;
+            if (bytes > OUTPUT_BYTE_LIMIT) {
+                child.kill();
+                return;
+            }
+            if (!stopAfterResultLimit) {
+                chunks.push(chunk);
+                return;
+            }
+            pending = Buffer.concat([pending, chunk]);
+            let newline = pending.indexOf(0x0a);
+            while (newline >= 0) {
+                const record = pending.subarray(0, newline + 1);
+                chunks.push(record);
+                if (record.includes(Buffer.from('"type":"match"')))
+                    matches += 1;
+                pending = pending.subarray(newline + 1);
+                if (matches > RESULT_LIMIT) {
+                    stopped = true;
+                    child.kill();
+                    return;
+                }
+                newline = pending.indexOf(0x0a);
+            }
+        });
+        child.once('close', (code, signal) => {
+            clearTimeout(timer);
+            if (stopped)
+                resolve({ code: 0, stdout: Buffer.concat(chunks) });
+            else if (signal !== null || bytes > OUTPUT_BYTE_LIMIT)
+                reject(new Error('CONTENT_SEARCH_COMMAND_LIMIT'));
+            else {
+                if (stopAfterResultLimit && pending.length > 0)
+                    chunks.push(pending);
+                resolve({ code: code ?? -1, stdout: Buffer.concat(chunks) });
+            }
+        });
+    });
+}
+function parseOutput(engine, output, query) {
+    if (engine === 'RIPGREP')
+        return parseRipgrep(output, query);
+    if (engine === 'GREP')
+        return parseGrep(output, query);
+    return parsePowerShell(output, query);
+}
+function parseRipgrep(output, query) {
+    return output
+        .toString('utf8')
+        .split('\n')
+        .flatMap((line) => {
+        if (line === '')
+            return [];
+        const value = JSON.parse(line);
+        if (value.type !== 'match')
+            return [];
+        const data = value.data;
+        const preview = String(data?.lines?.text).replace(/[\r\n]+$/u, '');
+        const column = lowerIndex(preview, query);
+        if (typeof data?.path?.text !== 'string' || !Number.isInteger(data.line_number) || column < 0)
+            throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+        return [
+            {
+                path: data.path.text,
+                line: data.line_number,
+                column: column + 1,
+                preview: preview.slice(0, 8192)
+            }
+        ];
+    });
+}
+function parseGrep(output, query) {
+    const matches = [];
+    let offset = 0;
+    while (offset < output.length) {
+        const separator = output.indexOf(0, offset);
+        const end = output.indexOf(10, separator + 1);
+        if (separator < 0 || end < 0)
+            throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+        const path = output.subarray(offset, separator).toString('utf8');
+        const record = output.subarray(separator + 1, end).toString('utf8');
+        const colon = record.indexOf(':');
+        const preview = record.slice(colon + 1).replace(/\r$/u, '');
+        const column = lowerIndex(preview, query);
+        if (colon < 1 || column < 0)
+            throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+        matches.push({
+            path,
+            line: Number(record.slice(0, colon)),
+            column: column + 1,
+            preview: preview.slice(0, 8192)
+        });
+        offset = end + 1;
+    }
+    return matches;
+}
+function parsePowerShell(output, query) {
+    return output
+        .toString('utf8')
+        .split(/\r?\n/u)
+        .flatMap((line) => {
+        if (line === '')
+            return [];
+        const match = JSON.parse(line);
+        if (typeof match.path !== 'string' ||
+            !Number.isInteger(match.line) ||
+            typeof match.preview !== 'string')
+            throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+        const column = lowerIndex(match.preview, query);
+        if (column < 0)
+            throw new Error('CONTENT_SEARCH_OUTPUT_INVALID');
+        return [
+            {
+                path: match.path,
+                line: match.line,
+                column: column + 1,
+                preview: match.preview.slice(0, 8192)
+            }
+        ];
+    });
+}
+async function nodeSearch(files, query, deadline) {
+    const matches = [];
+    let skippedFiles = 0;
+    for (const file of files) {
+        if (Date.now() >= deadline || matches.length > RESULT_LIMIT)
+            break;
+        const fileResults = [];
+        const stream = createReadStream(file.absolutePath);
+        const decoder = new TextDecoder('utf-8', { fatal: true });
+        let pending = '';
+        let line = 0;
+        let fileMatches = 0;
+        const addLine = (value) => {
+            line += 1;
+            const preview = value.endsWith('\r') ? value.slice(0, -1) : value;
+            const column = lowerIndex(preview, query);
+            if (column >= 0) {
+                fileResults.push({
+                    path: file.path,
+                    line,
+                    column: column + 1,
+                    preview: preview.slice(0, 8192)
+                });
+                fileMatches += 1;
+            }
+            return (fileMatches >= PER_FILE_LIMIT ||
+                matches.length + fileResults.length > RESULT_LIMIT ||
+                Date.now() >= deadline);
+        };
+        try {
+            let stopped = false;
+            for await (const chunk of stream) {
+                pending += decoder.decode(chunk, { stream: true });
+                let newline = pending.indexOf('\n');
+                while (newline >= 0) {
+                    if (addLine(pending.slice(0, newline))) {
+                        stopped = true;
+                        break;
+                    }
+                    pending = pending.slice(newline + 1);
+                    newline = pending.indexOf('\n');
+                }
+                if (stopped)
+                    break;
+            }
+            if (!stopped) {
+                pending += decoder.decode();
+                if (pending !== '')
+                    addLine(pending);
+            }
+            matches.push(...fileResults);
+        }
+        catch {
+            // Files that disappear, become unreadable or contain invalid UTF-8 are skipped atomically.
+            skippedFiles += 1;
+        }
+    }
+    return { matches, skippedFiles };
+}
+function lowerIndex(source, query) {
+    return source.toLocaleLowerCase().indexOf(query.toLocaleLowerCase());
+}
+function limitPerFile(matches) {
+    const counts = new Map();
+    return matches.filter((match) => {
+        const count = counts.get(match.path) ?? 0;
+        if (count >= PER_FILE_LIMIT)
+            return false;
+        counts.set(match.path, count + 1);
+        return true;
+    });
+}

package/dist/service/workspace-domain-state.d.ts

@@ -0,0 +1,75 @@
+import type { FileHandle } from 'node:fs/promises';
+import type { NodeAgentSession } from '../database.js';
+import type { WorkspaceFileIndex } from '../workspace.js';
+export interface WorkspaceWatch {
+    readonly workspaceId: string;
+    readonly watchId: string;
+    readonly sessionHash: string;
+    expiresAt: number;
+    readonly topics: {
+        readonly sessions: boolean;
+        readonly changes: boolean;
+        readonly files: {
+            readonly path: string;
+            readonly cursor?: string;
+            readonly limit?: number;
+        } | undefined;
+    };
+    timer: NodeJS.Timeout;
+}
+export interface ActiveWorkspaceUpload {
+    readonly id: string;
+    readonly workspaceId: string;
+    readonly path: string;
+    readonly targetPath: string;
+    readonly temporaryPath: string;
+    readonly size: number;
+    readonly overwrite: boolean;
+    readonly composerAttachment?: {
+        readonly id: string;
+        readonly name: string;
+        readonly mime: string;
+    };
+    readonly handle: FileHandle;
+    receivedBytes: number;
+    readonly timer: NodeJS.Timeout;
+}
+export interface CompletedComposerAttachment {
+    readonly id: string;
+    readonly workspaceId: string;
+    readonly path: string;
+    readonly targetPath: string;
+    readonly name: string;
+    readonly mime: string;
+    readonly size: number;
+}
+export declare class WorkspaceDomainState {
+    readonly uploads: Map<string, ActiveWorkspaceUpload>;
+    readonly composerAttachments: Map<string, CompletedComposerAttachment>;
+    readonly changesReads: Map<string, Promise<unknown>>;
+    readonly changesCache: Map<string, {
+        readonly value: unknown;
+        readonly expiresAt: number;
+    }>;
+    readonly changesVersions: Map<string, number>;
+    readonly fileIndexes: Map<string, WorkspaceFileIndex>;
+    readonly watches: Map<string, Map<string, WorkspaceWatch>>;
+    readonly watchRevisions: Map<string, number>;
+    readonly sessionWatchTimers: Map<string, NodeJS.Timeout>;
+    readonly sessionWatchSignatures: Map<string, string>;
+    readonly sessionWatchReads: Map<string, Promise<void>>;
+    readonly sessionCache: Map<string, {
+        readonly sessions: readonly NodeAgentSession[];
+        readonly expiresAt: number;
+    }>;
+    readonly sessionCacheReads: Map<string, Promise<readonly NodeAgentSession[]>>;
+    readonly changesLimiter: AsyncLimiter;
+}
+declare class AsyncLimiter {
+    private readonly limit;
+    private active;
+    private readonly waiters;
+    constructor(limit: number);
+    run<T>(task: () => Promise<T>): Promise<T>;
+}
+export {};

package/dist/service/workspace-domain-state.js

@@ -0,0 +1,40 @@
+export class WorkspaceDomainState {
+    uploads = new Map();
+    composerAttachments = new Map();
+    changesReads = new Map();
+    changesCache = new Map();
+    changesVersions = new Map();
+    fileIndexes = new Map();
+    watches = new Map();
+    watchRevisions = new Map();
+    sessionWatchTimers = new Map();
+    sessionWatchSignatures = new Map();
+    sessionWatchReads = new Map();
+    sessionCache = new Map();
+    sessionCacheReads = new Map();
+    changesLimiter = new AsyncLimiter(3);
+}
+class AsyncLimiter {
+    limit;
+    active = 0;
+    waiters = [];
+    constructor(limit) {
+        this.limit = limit;
+    }
+    async run(task) {
+        if (this.active < this.limit)
+            this.active += 1;
+        else
+            await new Promise((resolve) => this.waiters.push(resolve));
+        try {
+            return await task();
+        }
+        finally {
+            const next = this.waiters.shift();
+            if (next === undefined)
+                this.active -= 1;
+            else
+                next();
+        }
+    }
+}

package/dist/service/workspace-file-service.d.ts

@@ -0,0 +1,17 @@
+import type { NodeWorkspace } from '../database.js';
+import { WorkspaceFileIndex } from '../workspace.js';
+import { WorkspaceDomainState } from './workspace-domain-state.js';
+export declare class WorkspaceFileService {
+    private readonly state;
+    private readonly allowedRoots;
+    private readonly onInvalidated;
+    private readonly mutationTails;
+    constructor(state: WorkspaceDomainState, allowedRoots: () => readonly string[], onInvalidated: (workspaceId: string) => void);
+    list(workspace: NodeWorkspace, input: Record<string, unknown>): Promise<unknown>;
+    read(workspace: NodeWorkspace, input: Record<string, unknown>): Promise<unknown>;
+    search(workspace: NodeWorkspace, input: Record<string, unknown>): Promise<unknown>;
+    mutate(workspace: NodeWorkspace, input: Record<string, unknown>): Promise<unknown>;
+    private serializeMutation;
+    index(workspace: NodeWorkspace): WorkspaceFileIndex;
+    dispose(workspaceId?: string): void;
+}