@memorax/memorax-code 0.1.0

package/lib/memorax-code-backend/dist/incremental-jsonl-projection.js

@@ -0,0 +1,271 @@
+import { createHash } from "node:crypto";
+import { open, readFile, readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+const projections = new Map();
+const HASH_CHUNK_BYTES = 64 * 1024;
+export async function readIncrementalJsonlProjection(options) {
+    return (await readIncrementalJsonlProjectionSnapshot(options)).values;
+}
+export async function readIncrementalJsonlProjectionSnapshot(options) {
+    const projectionKey = options.namespace
+        ? `${options.namespace}\u0000${options.root}\u0000${options.filename}`
+        : `${options.root}\u0000${options.filename}`;
+    let projection = projections.get(projectionKey);
+    if (projection && projection.identity !== options.identity)
+        projection = undefined;
+    if (!projection) {
+        projection = { identity: options.identity, files: new Map(), values: [] };
+        projections.set(projectionKey, projection);
+    }
+    if (projection.refresh)
+        return projection.refresh;
+    const refresh = refreshProjection(projection, options);
+    projection.refresh = refresh;
+    try {
+        return await refresh;
+    }
+    finally {
+        if (projection.refresh === refresh)
+            projection.refresh = undefined;
+    }
+}
+export function clearIncrementalJsonlProjections() {
+    projections.clear();
+}
+async function refreshProjection(projection, options) {
+    let directories;
+    try {
+        directories = (await readdir(options.root, { withFileTypes: true }))
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => entry.name);
+    }
+    catch (error) {
+        if (projection.files.size > 0 || projection.values.length > 0) {
+            projection.files.clear();
+            projection.values = [];
+        }
+        return {
+            values: projection.values,
+            complete: error.code === "ENOENT",
+        };
+    }
+    const activeDirectories = new Set(directories);
+    let rebuild = false;
+    let complete = true;
+    for (const cachedDirectory of projection.files.keys()) {
+        if (activeDirectories.has(cachedDirectory))
+            continue;
+        projection.files.delete(cachedDirectory);
+        rebuild = true;
+    }
+    if (options.refreshEntry) {
+        for (const file of projection.files.values()) {
+            const retryEntries = [];
+            for (const entry of file.retryEntries) {
+                if (options.refreshEntry(entry))
+                    rebuild = true;
+                if (entry.retryKey)
+                    retryEntries.push(entry);
+            }
+            file.retryEntries = retryEntries;
+        }
+    }
+    const appended = [];
+    for (const directory of directories) {
+        const path = join(options.root, directory, options.filename);
+        const cached = projection.files.get(directory);
+        let metadata;
+        try {
+            metadata = await stat(path);
+        }
+        catch (error) {
+            const missing = error.code === "ENOENT";
+            if (cached && missing) {
+                projection.files.delete(directory);
+                rebuild = true;
+            }
+            if (!missing)
+                complete = false;
+            continue;
+        }
+        if (!metadata.isFile()) {
+            complete = false;
+            continue;
+        }
+        if (cached
+            && metadata.size === cached.byteLength
+            && metadata.mtimeMs === cached.modifiedAtMs
+            && metadata.ctimeMs === cached.changedAtMs
+            && metadata.ino === cached.inode) {
+            if (!cached.complete)
+                complete = false;
+            continue;
+        }
+        let buffer;
+        let verifiedPrefixHash;
+        let rangeStart;
+        let appendOnly = false;
+        try {
+            if (cached && metadata.ino === cached.inode && metadata.size > cached.byteLength) {
+                verifiedPrefixHash = await hashFilePrefix(path, cached.byteLength);
+                appendOnly = verifiedPrefixHash.copy().digest("hex") === cached.contentHash;
+                if (appendOnly) {
+                    buffer = await readFileRange(path, cached.byteLength, metadata.size);
+                    rangeStart = cached.byteLength;
+                }
+                else {
+                    buffer = await readFile(path);
+                    rangeStart = 0;
+                }
+            }
+            else {
+                buffer = await readFile(path);
+                rangeStart = 0;
+            }
+        }
+        catch {
+            complete = false;
+            continue;
+        }
+        const parsed = parseJsonlChunk(buffer, directory, options.project);
+        const readComplete = buffer.length === metadata.size - rangeStart;
+        const consumedComplete = (appendOnly ? cached.consumedComplete : true)
+            && parsed.consumedComplete
+            && readComplete;
+        const fileComplete = (appendOnly ? cached.consumedComplete : true)
+            && parsed.complete
+            && readComplete;
+        if (!fileComplete)
+            complete = false;
+        const entries = appendOnly ? [...cached.entries, ...parsed.entries] : parsed.entries;
+        const parsedRetryEntries = parsed.entries.filter((entry) => entry.retryKey);
+        const retryEntries = appendOnly
+            ? [...cached.retryEntries, ...parsedRetryEntries]
+            : parsedRetryEntries;
+        const byteLength = rangeStart + parsed.consumedByteLength;
+        projection.files.set(directory, {
+            byteLength,
+            modifiedAtMs: metadata.mtimeMs,
+            changedAtMs: metadata.ctimeMs,
+            inode: metadata.ino,
+            contentHash: appendOnly
+                ? verifiedPrefixHash.update(buffer.subarray(0, parsed.consumedByteLength)).digest("hex")
+                : hashBuffer(buffer.subarray(0, parsed.consumedByteLength)),
+            complete: fileComplete,
+            consumedComplete,
+            entries,
+            retryEntries,
+        });
+        if (appendOnly || !cached)
+            appended.push(...parsed.entries.map(({ value }) => value));
+        else
+            rebuild = true;
+    }
+    if (rebuild) {
+        projection.values = [...projection.files.values()]
+            .flatMap((file) => file.entries.map(({ value }) => value))
+            .sort(options.compare);
+    }
+    else if (appended.length > 0) {
+        const chronological = appended.sort(options.compare);
+        const last = projection.values.at(-1);
+        projection.values = !last || options.compare(last, chronological[0]) <= 0
+            ? [...projection.values, ...chronological]
+            : [...projection.values, ...chronological].sort(options.compare);
+    }
+    return { values: projection.values, complete };
+}
+async function readFileRange(path, start, end) {
+    const handle = await open(path, "r");
+    try {
+        const buffer = Buffer.allocUnsafe(end - start);
+        let offset = 0;
+        while (offset < buffer.length) {
+            const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, start + offset);
+            if (bytesRead === 0)
+                break;
+            offset += bytesRead;
+        }
+        return buffer.subarray(0, offset);
+    }
+    finally {
+        await handle.close();
+    }
+}
+function parseJsonlChunk(buffer, directory, project) {
+    const entries = [];
+    let consumedByteLength = 0;
+    let complete = true;
+    let consumedComplete = true;
+    let lineStart = 0;
+    while (lineStart < buffer.length) {
+        const newline = buffer.indexOf(0x0a, lineStart);
+        const lineComplete = newline !== -1;
+        const lineEnd = lineComplete ? newline : buffer.length;
+        const line = buffer.subarray(lineStart, lineEnd).toString("utf8");
+        if (lineComplete)
+            consumedByteLength = lineEnd + 1;
+        if (!line.trim()) {
+            if (!lineComplete)
+                consumedByteLength = buffer.length;
+            lineStart = lineEnd + 1;
+            continue;
+        }
+        let value;
+        try {
+            value = JSON.parse(line);
+        }
+        catch {
+            complete = false;
+            if (lineComplete)
+                consumedComplete = false;
+            if (!lineComplete)
+                break;
+            lineStart = lineEnd + 1;
+            continue;
+        }
+        if (!lineComplete)
+            consumedByteLength = buffer.length;
+        try {
+            const entry = project(value, directory);
+            if (entry)
+                entries.push(entry);
+        }
+        catch {
+            complete = false;
+            consumedComplete = false;
+        }
+        if (!lineComplete)
+            break;
+        lineStart = lineEnd + 1;
+    }
+    return {
+        entries,
+        consumedByteLength,
+        complete,
+        consumedComplete,
+    };
+}
+async function hashFilePrefix(path, end) {
+    const handle = await open(path, "r");
+    const hash = createHash("sha256");
+    try {
+        const buffer = Buffer.allocUnsafe(Math.min(HASH_CHUNK_BYTES, end));
+        let position = 0;
+        while (position < end) {
+            const length = Math.min(buffer.length, end - position);
+            const { bytesRead } = await handle.read(buffer, 0, length, position);
+            if (bytesRead === 0)
+                break;
+            hash.update(buffer.subarray(0, bytesRead));
+            position += bytesRead;
+        }
+        return hash;
+    }
+    finally {
+        await handle.close();
+    }
+}
+function hashBuffer(buffer) {
+    return createHash("sha256").update(buffer).digest("hex");
+}

package/lib/memorax-code-backend/dist/install-watchdog.js

@@ -0,0 +1,71 @@
+import { existsSync } from "node:fs";
+import { delimiter } from "node:path";
+export function resolveBackendInstallWatchdogConfig(overrides = {}) {
+    const envPaths = nonEmptyString(process.env.MEMORAX_CODE_INSTALL_WATCH_PATHS)
+        ?.split(delimiter)
+        .map((item) => item.trim())
+        .filter(Boolean);
+    const paths = overrides.paths ?? envPaths;
+    if (!paths || paths.length === 0)
+        return undefined;
+    return {
+        enabled: overrides.enabled ?? parseBooleanEnv(process.env.MEMORAX_CODE_INSTALL_WATCHDOG) ?? true,
+        paths,
+        intervalMs: clampPositiveInteger(overrides.intervalMs ?? Number(process.env.MEMORAX_CODE_INSTALL_WATCH_INTERVAL_MS), 2000),
+        graceMs: clampPositiveInteger(overrides.graceMs ?? Number(process.env.MEMORAX_CODE_INSTALL_WATCH_GRACE_MS), 10000),
+        exitProcess: overrides.exitProcess ?? parseBooleanEnv(process.env.MEMORAX_CODE_INSTALL_WATCH_EXIT) ?? true,
+    };
+}
+export function startBackendInstallWatchdog(config, onShutdown) {
+    if (!config?.enabled || config.paths.length === 0)
+        return undefined;
+    let missingSince;
+    let triggered = false;
+    const check = () => {
+        if (triggered)
+            return;
+        const missingPaths = missingInstallPaths(config);
+        if (missingPaths.length === 0) {
+            missingSince = undefined;
+            return;
+        }
+        const now = Date.now();
+        missingSince ??= now;
+        if (now - missingSince < config.graceMs)
+            return;
+        triggered = true;
+        clearInterval(timer);
+        void onShutdown({ reason: "install_missing", missingPaths });
+    };
+    const timer = setInterval(check, config.intervalMs);
+    timer.unref?.();
+    check();
+    return {
+        close() {
+            triggered = true;
+            clearInterval(timer);
+        },
+    };
+}
+function missingInstallPaths(config) {
+    return config.paths.filter((path) => !existsSync(path));
+}
+function nonEmptyString(value) {
+    return typeof value === "string" && value.trim() ? value : undefined;
+}
+function clampPositiveInteger(value, fallback) {
+    const number = typeof value === "number" ? value : fallback;
+    if (!Number.isFinite(number))
+        return fallback;
+    return Math.max(1, Math.floor(number));
+}
+function parseBooleanEnv(value) {
+    if (value === undefined)
+        return undefined;
+    const normalized = value.trim().toLowerCase();
+    if (["1", "true", "yes", "on", "enabled"].includes(normalized))
+        return true;
+    if (["0", "false", "no", "off", "disabled"].includes(normalized))
+        return false;
+    return undefined;
+}

package/lib/memorax-code-backend/dist/jsonl-append.js

@@ -0,0 +1,75 @@
+import { open } from "node:fs/promises";
+const JSONL_TAIL_SCAN_CHUNK_BYTES = 8 * 1024;
+const pathOperations = new Map();
+export async function appendRepairingJsonl(path, text) {
+    await serializePath(path, async () => {
+        const handle = await open(path, "a+", 0o600);
+        try {
+            const { rollbackSize, separator } = await prepareTailForAppend(handle);
+            try {
+                await handle.writeFile(`${separator}${text}`, "utf8");
+            }
+            catch (error) {
+                await handle.truncate(rollbackSize).catch(() => undefined);
+                throw error;
+            }
+        }
+        finally {
+            await handle.close();
+        }
+    });
+}
+async function serializePath(path, operation) {
+    const previous = pathOperations.get(path) ?? Promise.resolve();
+    const current = previous.catch(() => undefined).then(operation);
+    pathOperations.set(path, current);
+    try {
+        return await current;
+    }
+    finally {
+        if (pathOperations.get(path) === current)
+            pathOperations.delete(path);
+    }
+}
+async function prepareTailForAppend(handle) {
+    const size = (await handle.stat()).size;
+    if (size === 0)
+        return { rollbackSize: 0, separator: "" };
+    const lastByte = Buffer.allocUnsafe(1);
+    await readExactAt(handle, lastByte, size - 1);
+    if (lastByte[0] === 0x0a)
+        return { rollbackSize: size, separator: "" };
+    const tailStart = await jsonlTailStart(handle, size);
+    const tail = Buffer.allocUnsafe(size - tailStart);
+    await readExactAt(handle, tail, tailStart);
+    try {
+        JSON.parse(tail.toString("utf8"));
+        return { rollbackSize: size, separator: "\n" };
+    }
+    catch {
+        await handle.truncate(tailStart);
+        return { rollbackSize: tailStart, separator: "" };
+    }
+}
+async function jsonlTailStart(handle, size) {
+    let end = size;
+    while (end > 0) {
+        const start = Math.max(0, end - JSONL_TAIL_SCAN_CHUNK_BYTES);
+        const buffer = Buffer.allocUnsafe(end - start);
+        await readExactAt(handle, buffer, start);
+        const newline = buffer.lastIndexOf(0x0a);
+        if (newline >= 0)
+            return start + newline + 1;
+        end = start;
+    }
+    return 0;
+}
+async function readExactAt(handle, buffer, position) {
+    let offset = 0;
+    while (offset < buffer.length) {
+        const { bytesRead } = await handle.read(buffer, offset, buffer.length - offset, position + offset);
+        if (bytesRead === 0)
+            throw new Error("failed to read JSONL tail");
+        offset += bytesRead;
+    }
+}