@openclaw/crabline 0.1.9 → 0.1.10

package/dist/src/providers/recorder.js

@@ -1,12 +1,372 @@
-import { appendFile, mkdir, open, readFile } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, readlink, realpath } from "node:fs/promises";
 import path from "node:path";
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+import { lock } from "proper-lockfile";
+function sleep(ms, signal) {
+    if (signal?.aborted) {
+        return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+        const done = () => {
+            clearTimeout(timeout);
+            signal?.removeEventListener("abort", done);
+            resolve();
+        };
+        const timeout = setTimeout(done, ms);
+        signal?.addEventListener("abort", done, { once: true });
+    });
+}
+const pendingAppends = new Map();
+const recordIdentityIndexes = new Map();
+const MAX_RECORD_IDENTITY_INDEXES = 128;
+const MAX_RECENT_RECORD_KEYS = 4096;
+const RECORDER_BATCH_VERSION = 1;
+const RECORDER_LOCK_STALE_MS = 30_000;
+const RECORDER_LOCK_UPDATE_MS = 10_000;
+const RECORDER_ROTATION_ATTEMPTS = 3;
+class RecorderRotatedError extends Error {
 }
-function toRecordKey(event) {
-    return `${event.provider}:${event.threadId}:${event.id}`;
+const CONTINUITY_BYTES = 4096;
+const MAX_INCREMENTAL_READ_BYTES = 256 * 1024;
+const MAX_PENDING_RECORD_BYTES = 4 * 1024 * 1024;
+function createIncrementalReadState() {
+    return {
+        caughtUp: true,
+        continuity: Buffer.alloc(0),
+        generation: 0,
+        identity: undefined,
+        offset: 0,
+        pending: Buffer.alloc(0),
+    };
+}
+export function createRecordedInboundCursor() {
+    return {
+        buffered: [],
+        readState: createIncrementalReadState(),
+        seen: new Set(),
+    };
+}
+export function cloneRecordedInboundCursor(cursor) {
+    return {
+        buffered: [...cursor.buffered],
+        readState: {
+            caughtUp: cursor.readState.caughtUp,
+            continuity: Buffer.from(cursor.readState.continuity),
+            generation: cursor.readState.generation,
+            identity: cursor.readState.identity ? { ...cursor.readState.identity } : undefined,
+            offset: cursor.readState.offset,
+            pending: Buffer.from(cursor.readState.pending),
+        },
+        seen: new Set(cursor.seen),
+    };
+}
+function rememberRecentRecord(seen, event) {
+    const key = JSON.stringify([
+        event.provider,
+        event.threadId,
+        event.id,
+        recordedDirectionOf(event),
+    ]);
+    if (seen.has(key)) {
+        return false;
+    }
+    seen.add(key);
+    if (seen.size > MAX_RECENT_RECORD_KEYS) {
+        seen.delete(seen.values().next().value);
+    }
+    return true;
+}
+function recordedDirectionOf(event) {
+    if (event.recordedDirection) {
+        return event.recordedDirection;
+    }
+    return event.raw !== null &&
+        typeof event.raw === "object" &&
+        "direction" in event.raw &&
+        event.raw.direction === "outbound"
+        ? "outbound"
+        : "inbound";
+}
+function requireNonEmptyString(value, field) {
+    if (typeof value !== "string" || value.length === 0) {
+        throw new Error(`Recorded inbound envelope ${field} must be a non-empty string.`);
+    }
+    return value;
+}
+function requireString(value, field) {
+    if (typeof value !== "string") {
+        throw new Error(`Recorded inbound envelope ${field} must be a string.`);
+    }
+    return value;
+}
+function parseRecordedEnvelope(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        throw new Error("Recorded inbound envelope must be an object.");
+    }
+    const record = value;
+    if (record.author !== "assistant" && record.author !== "system" && record.author !== "user") {
+        throw new Error("Recorded inbound envelope author must be assistant, system, or user.");
+    }
+    if (record.recordedDirection !== undefined &&
+        record.recordedDirection !== "inbound" &&
+        record.recordedDirection !== "outbound") {
+        throw new Error("Recorded inbound envelope recordedDirection must be inbound or outbound.");
+    }
+    return {
+        author: record.author,
+        id: requireNonEmptyString(record.id, "id"),
+        provider: requireNonEmptyString(record.provider, "provider"),
+        ...(record.raw !== undefined ? { raw: record.raw } : {}),
+        recordedAt: requireNonEmptyString(record.recordedAt, "recordedAt"),
+        ...(record.recordedDirection !== undefined
+            ? { recordedDirection: record.recordedDirection }
+            : {}),
+        sentAt: requireNonEmptyString(record.sentAt, "sentAt"),
+        text: requireString(record.text, "text"),
+        threadId: requireNonEmptyString(record.threadId, "threadId"),
+    };
+}
+function parseRecordedLine(line) {
+    const parsed = JSON.parse(line);
+    if (typeof parsed === "object" &&
+        parsed !== null &&
+        !Array.isArray(parsed) &&
+        "recordType" in parsed &&
+        parsed.recordType === "crabline.recorder.batch" &&
+        "recorderBatchVersion" in parsed &&
+        parsed.recorderBatchVersion === RECORDER_BATCH_VERSION &&
+        "events" in parsed &&
+        Array.isArray(parsed.events)) {
+        return parsed.events.map(parseRecordedEnvelope);
+    }
+    return [parseRecordedEnvelope(parsed)];
+}
+function resetIncrementalReadState(state) {
+    if (state.identity !== undefined ||
+        state.offset > 0 ||
+        state.pending.length > 0 ||
+        state.continuity.length > 0) {
+        state.generation += 1;
+    }
+    state.caughtUp = true;
+    state.continuity = Buffer.alloc(0);
+    state.offset = 0;
+    state.pending = Buffer.alloc(0);
+}
+async function readBufferAt(handle, length, position) {
+    const buffer = Buffer.alloc(length);
+    let bytesRead = 0;
+    while (bytesRead < length) {
+        const result = await handle.read(buffer, bytesRead, length - bytesRead, position + bytesRead);
+        if (result.bytesRead === 0) {
+            break;
+        }
+        bytesRead += result.bytesRead;
+    }
+    return buffer.subarray(0, bytesRead);
+}
+function consumeRecordedChunk(state, chunk) {
+    const continuity = Buffer.concat([state.continuity, chunk]);
+    state.continuity = Buffer.from(continuity.subarray(-CONTINUITY_BYTES));
+    const raw = state.pending.length > 0 ? Buffer.concat([state.pending, chunk]) : chunk;
+    const lastNewline = raw.lastIndexOf(0x0a);
+    if (lastNewline < 0) {
+        if (raw.length > MAX_PENDING_RECORD_BYTES) {
+            throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
+        }
+        state.pending = Buffer.from(raw);
+        return [];
+    }
+    state.pending = Buffer.from(raw.subarray(lastNewline + 1));
+    if (state.pending.length > MAX_PENDING_RECORD_BYTES) {
+        throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
+    }
+    const events = [];
+    for (const line of raw.subarray(0, lastNewline).toString("utf8").split("\n")) {
+        if (line.trim()) {
+            events.push(...parseRecordedLine(line));
+        }
+    }
+    return events;
+}
+async function appendJsonLine(filePath, line) {
+    await serializeAppend(filePath, async (publicationPath, logicalPath) => {
+        await appendCommittedLine(publicationPath, logicalPath, line, true);
+    });
+}
+async function readRecorderFileIdentity(filePath) {
+    try {
+        const stats = await lstat(filePath, { bigint: true });
+        if (!stats.isFile()) {
+            throw new Error(`Recorder path is not a regular file: ${filePath}`);
+        }
+        return {
+            dev: stats.dev,
+            ino: stats.ino,
+        };
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return undefined;
+        }
+        throw error;
+    }
+}
+function sameRecorderFileIdentity(left, right) {
+    return left?.dev === right?.dev && left?.ino === right?.ino;
+}
+async function resolveRecorderPublicationPath(filePath) {
+    try {
+        return await realpath(filePath);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    try {
+        const stats = await lstat(filePath);
+        if (stats.isSymbolicLink()) {
+            const target = await readlink(filePath);
+            return await resolveRecorderPublicationPath(path.resolve(path.dirname(filePath), target));
+        }
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    return path.join(await realpath(path.dirname(filePath)), path.basename(filePath));
+}
+async function prepareRecorderTailForAppend(handle) {
+    const stats = await handle.stat();
+    if (stats.size === 0) {
+        return false;
+    }
+    const finalByte = await readBufferAt(handle, 1, stats.size - 1);
+    if (finalByte[0] === 0x0a) {
+        return false;
+    }
+    const windowSize = Math.min(stats.size, MAX_PENDING_RECORD_BYTES + 1);
+    const tailWindow = await readBufferAt(handle, windowSize, stats.size - windowSize);
+    const lastNewline = tailWindow.lastIndexOf(0x0a);
+    if (lastNewline < 0 && stats.size > MAX_PENDING_RECORD_BYTES) {
+        throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
+    }
+    const tailStart = stats.size - windowSize + lastNewline + 1;
+    const tail = tailWindow.subarray(lastNewline + 1).toString("utf8");
+    try {
+        JSON.parse(tail);
+        await handle.writeFile("\n", "utf8");
+        return true;
+    }
+    catch (error) {
+        if (!(error instanceof SyntaxError)) {
+            throw error;
+        }
+        await handle.truncate(tailStart);
+        return true;
+    }
+}
+async function prepareRecorderPathForAppend(publicationPath, logicalPath, durable) {
+    const handle = await open(publicationPath, "a+");
+    const identity = await handle.stat({ bigint: true });
+    const recorderIdentity = { dev: identity.dev, ino: identity.ino };
+    try {
+        const changed = await prepareRecorderTailForAppend(handle);
+        if (changed && durable) {
+            await handle.sync();
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
+        throw new RecorderRotatedError("Recorder rotated while preparing a committed line.");
+    }
+    return recorderIdentity;
+}
+async function appendCommittedLine(publicationPath, logicalPath, line, durable, expectedIdentity) {
+    const handle = await open(publicationPath, "a+");
+    const identity = await handle.stat({ bigint: true });
+    const recorderIdentity = { dev: identity.dev, ino: identity.ino };
+    try {
+        if (expectedIdentity !== undefined &&
+            !sameRecorderFileIdentity(expectedIdentity, recorderIdentity)) {
+            throw new RecorderRotatedError("Recorder rotated before appending a committed line.");
+        }
+        await prepareRecorderTailForAppend(handle);
+        await handle.writeFile(line, "utf8");
+        if (durable) {
+            await handle.sync();
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    if (!sameRecorderFileIdentity(recorderIdentity, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
+        throw new RecorderRotatedError("Recorder rotated while appending a committed line.");
+    }
+}
+function recorderLockReleaseError(filePath, operationError, releaseError) {
+    return new AggregateError([operationError, releaseError], `Recorder append and lock release both failed for "${filePath}".`, { cause: operationError });
+}
+async function withRecorderLock(filePath, operation) {
+    const release = await lock(filePath, {
+        realpath: false,
+        retries: {
+            factor: 1,
+            maxTimeout: 10,
+            minTimeout: 10,
+            retries: 500,
+        },
+        stale: RECORDER_LOCK_STALE_MS,
+        update: RECORDER_LOCK_UPDATE_MS,
+    });
+    let operationFailed = false;
+    let operationError;
+    let result;
+    try {
+        result = await operation();
+    }
+    catch (error) {
+        operationFailed = true;
+        operationError = error;
+    }
+    try {
+        await release();
+    }
+    catch (releaseError) {
+        if (operationFailed) {
+            throw recorderLockReleaseError(filePath, operationError, releaseError);
+        }
+        throw releaseError;
+    }
+    if (operationFailed) {
+        throw operationError;
+    }
+    return result;
+}
+async function serializeAppend(filePath, operation) {
+    const logicalPath = path.resolve(filePath);
+    const key = await resolveRecorderPublicationPath(logicalPath);
+    const previous = pendingAppends.get(key) ?? Promise.resolve();
+    let result;
+    const current = previous
+        .catch(() => { })
+        .then(async () => {
+        result = await withRecorderLock(key, async () => await operation(key, logicalPath));
+    });
+    pendingAppends.set(key, current);
+    try {
+        await current;
+        return result;
+    }
+    finally {
+        if (pendingAppends.get(key) === current) {
+            pendingAppends.delete(key);
+        }
+    }
 }
-const MAX_WATCH_SEEN_KEYS = 4096;
 async function readRecordedInboundAppend(filePath, state) {
     let handle;
     try {
@@ -15,54 +375,49 @@ async function readRecordedInboundAppend(filePath, state) {
     catch (error) {
         const code = error.code;
         if (code === "ENOENT") {
-            state.offset = 0;
-            state.pending = Buffer.alloc(0);
+            resetIncrementalReadState(state);
+            state.identity = undefined;
             return [];
         }
         throw error;
     }
     try {
-        const size = (await handle.stat()).size;
-        if (size < state.offset) {
-            state.offset = 0;
-            state.pending = Buffer.alloc(0);
+        const stats = await handle.stat();
+        const identity = { dev: stats.dev, ino: stats.ino };
+        const sameFile = state.identity?.dev === identity.dev && state.identity.ino === identity.ino;
+        const rotated = state.identity !== undefined && !sameFile;
+        let hasContinuity = stats.size >= state.offset;
+        if (hasContinuity && state.offset > 0 && state.continuity.length === 0) {
+            hasContinuity = sameFile;
         }
-        const chunks = [];
+        else if (hasContinuity && state.continuity.length > 0) {
+            const actual = await readBufferAt(handle, state.continuity.length, state.offset - state.continuity.length);
+            hasContinuity = actual.equals(state.continuity);
+        }
+        if (!hasContinuity) {
+            resetIncrementalReadState(state);
+        }
+        else if (rotated) {
+            state.generation += 1;
+        }
+        state.identity = identity;
+        const events = [];
         let position = state.offset;
-        while (position < size) {
-            const chunk = Buffer.alloc(Math.min(64 * 1024, size - position));
+        let remainingBatchBytes = MAX_INCREMENTAL_READ_BYTES;
+        let reachedUnexpectedEof = false;
+        while (position < stats.size && remainingBatchBytes > 0) {
+            const chunk = Buffer.alloc(Math.min(64 * 1024, stats.size - position, remainingBatchBytes));
             const { bytesRead } = await handle.read(chunk, 0, chunk.length, position);
             if (bytesRead === 0) {
+                reachedUnexpectedEof = true;
                 break;
             }
-            chunks.push(chunk.subarray(0, bytesRead));
             position += bytesRead;
+            state.offset = position;
+            remainingBatchBytes -= bytesRead;
+            events.push(...consumeRecordedChunk(state, chunk.subarray(0, bytesRead)));
         }
-        state.offset = position;
-        const raw = Buffer.concat([state.pending, ...chunks]);
-        const lastNewline = raw.lastIndexOf(0x0a);
-        if (lastNewline < 0) {
-            state.pending = raw;
-            return [];
-        }
-        state.pending = raw.subarray(lastNewline + 1);
-        const lines = raw
-            .subarray(0, lastNewline)
-            .toString("utf8")
-            .split("\n")
-            .filter((line) => line.trim().length > 0);
-        const events = [];
-        for (const [index, line] of lines.entries()) {
-            try {
-                events.push(JSON.parse(line));
-            }
-            catch (error) {
-                if (index === lines.length - 1 && state.pending.length === 0) {
-                    continue;
-                }
-                throw error;
-            }
-        }
+        state.caughtUp = reachedUnexpectedEof || position >= stats.size;
         return events;
     }
     finally {
@@ -75,9 +430,92 @@ export async function appendRecordedInbound(filePath, event) {
         ...event,
         recordedAt: new Date().toISOString(),
     };
-    await appendFile(filePath, `${JSON.stringify(recorded)}\n`, "utf8");
+    const line = `${JSON.stringify(recorded)}\n`;
+    if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
+        throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
+    }
+    await appendJsonLine(filePath, line);
     return recorded;
 }
+export async function appendRecordedInboundBatch(filePath, events) {
+    if (events.length === 0) {
+        return [];
+    }
+    await mkdir(path.dirname(filePath), { recursive: true });
+    for (let attempt = 0;; attempt++) {
+        try {
+            return await serializeAppend(filePath, async (publicationPath, logicalPath) => {
+                const generation = await prepareRecorderPathForAppend(publicationPath, logicalPath, true);
+                const seen = await syncRecordIdentityIndex(publicationPath);
+                const pendingIdentities = new Set();
+                const recorded = [];
+                for (const event of events) {
+                    const identity = recordIdentity(event);
+                    if (seen.has(identity) || pendingIdentities.has(identity)) {
+                        continue;
+                    }
+                    pendingIdentities.add(identity);
+                    recorded.push({
+                        ...event,
+                        recordedAt: new Date().toISOString(),
+                    });
+                }
+                if (recorded.length > 0) {
+                    const batch = {
+                        events: recorded,
+                        recordType: "crabline.recorder.batch",
+                        recorderBatchVersion: RECORDER_BATCH_VERSION,
+                    };
+                    const line = `${JSON.stringify(batch)}\n`;
+                    if (Buffer.byteLength(line) > MAX_PENDING_RECORD_BYTES) {
+                        throw new Error(`Recorder record exceeded ${MAX_PENDING_RECORD_BYTES} bytes without a newline.`);
+                    }
+                    await appendCommittedLine(publicationPath, logicalPath, line, true, generation);
+                    await syncRecordIdentityIndex(publicationPath);
+                }
+                else if (!sameRecorderFileIdentity(generation, await readRecorderFileIdentity(await resolveRecorderPublicationPath(logicalPath)))) {
+                    throw new RecorderRotatedError("Recorder rotated before confirming a duplicate batch.");
+                }
+                return recorded;
+            });
+        }
+        catch (error) {
+            if (!(error instanceof RecorderRotatedError) || attempt + 1 >= RECORDER_ROTATION_ATTEMPTS) {
+                throw error;
+            }
+        }
+    }
+}
+function recordIdentity(event) {
+    return JSON.stringify([event.provider, event.threadId, event.id, recordedDirectionOf(event)]);
+}
+async function syncRecordIdentityIndex(filePath) {
+    const key = path.resolve(filePath);
+    let index = recordIdentityIndexes.get(key);
+    if (!index) {
+        index = { readState: createIncrementalReadState(), seen: new Set() };
+        recordIdentityIndexes.set(key, index);
+        if (recordIdentityIndexes.size > MAX_RECORD_IDENTITY_INDEXES) {
+            recordIdentityIndexes.delete(recordIdentityIndexes.keys().next().value);
+        }
+    }
+    else {
+        recordIdentityIndexes.delete(key);
+        recordIdentityIndexes.set(key, index);
+    }
+    let generation = index.readState.generation;
+    do {
+        const appended = await readRecordedInboundAppend(filePath, index.readState);
+        if (index.readState.generation !== generation) {
+            index.seen.clear();
+            generation = index.readState.generation;
+        }
+        for (const event of appended) {
+            rememberRecentRecord(index.seen, event);
+        }
+    } while (!index.readState.caughtUp);
+    return index.seen;
+}
 export async function readRecordedInbound(filePath) {
     let raw = "";
     try {
@@ -90,62 +528,90 @@ export async function readRecordedInbound(filePath) {
         }
         throw error;
     }
-    const lines = raw.split("\n").filter((line) => line.trim().length > 0);
+    const lastNewline = raw.lastIndexOf("\n");
+    const completed = lastNewline >= 0 ? raw.slice(0, lastNewline) : "";
+    const tail = raw.slice(lastNewline + 1);
     const events = [];
-    for (const [index, line] of lines.entries()) {
+    for (const line of completed.split("\n")) {
+        if (line.trim()) {
+            events.push(...parseRecordedLine(line));
+        }
+    }
+    if (tail.trim()) {
         try {
-            events.push(JSON.parse(line));
+            events.push(...parseRecordedLine(tail));
         }
         catch (error) {
-            if (index === lines.length - 1) {
-                continue;
+            if (!(error instanceof SyntaxError)) {
+                throw error;
             }
-            throw error;
+            // Ignore a syntactically partial final append; completed lines remain strict.
         }
     }
     return events;
 }
 export async function waitForRecordedInbound(params) {
     const deadline = Date.now() + params.timeoutMs;
-    const seen = new Set();
-    while (Date.now() <= deadline) {
-        const events = await readRecordedInbound(params.filePath);
-        for (const event of events) {
-            const key = toRecordKey(event);
-            if (seen.has(key)) {
+    const cursor = params.cursor ?? createRecordedInboundCursor();
+    while (!params.signal?.aborted && Date.now() <= deadline) {
+        const generation = cursor.readState.generation;
+        const events = cursor.buffered.length > 0
+            ? cursor.buffered.splice(0)
+            : await readRecordedInboundAppend(params.filePath, cursor.readState);
+        if (cursor.readState.generation !== generation) {
+            cursor.seen.clear();
+        }
+        for (const [index, event] of events.entries()) {
+            if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
+                continue;
+            }
+            // Incremental read state owns progress; this bounded window only filters appended retries.
+            if (!rememberRecentRecord(cursor.seen, event)) {
                 continue;
             }
-            seen.add(key);
             if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
                 continue;
             }
             if (params.matches(event)) {
+                cursor.buffered.push(...events.slice(index + 1));
                 return event;
             }
         }
-        await sleep(params.pollMs ?? 200);
+        if (!cursor.readState.caughtUp) {
+            if (Date.now() >= deadline) {
+                return null;
+            }
+            continue;
+        }
+        const remainingMs = deadline - Date.now();
+        if (remainingMs <= 0) {
+            return null;
+        }
+        await sleep(Math.min(params.pollMs ?? 200, remainingMs), params.signal);
     }
     return null;
 }
 export async function* watchRecordedInbound(params) {
-    const state = {
-        offset: 0,
-        pending: Buffer.alloc(0),
-    };
+    const state = createIncrementalReadState();
     const seen = new Set();
-    while (true) {
+    while (!params.signal?.aborted) {
+        const generation = state.generation;
         const events = await readRecordedInboundAppend(params.filePath, state);
+        if (state.generation !== generation) {
+            seen.clear();
+        }
+        if (params.signal?.aborted) {
+            return;
+        }
         for (const event of events) {
-            const key = toRecordKey(event);
-            if (seen.has(key)) {
+            if (params.signal?.aborted) {
+                return;
+            }
+            if (!rememberRecentRecord(seen, event)) {
                 continue;
             }
-            seen.add(key);
-            if (seen.size > MAX_WATCH_SEEN_KEYS) {
-                const oldest = seen.values().next().value;
-                if (oldest !== undefined) {
-                    seen.delete(oldest);
-                }
+            if (params.recordedDirection && recordedDirectionOf(event) !== params.recordedDirection) {
+                continue;
             }
             if (params.since && new Date(event.sentAt).getTime() < new Date(params.since).getTime()) {
                 continue;
@@ -154,7 +620,10 @@ export async function* watchRecordedInbound(params) {
                 yield event;
             }
         }
-        await sleep(params.pollMs ?? 250);
+        if (!state.caughtUp) {
+            continue;
+        }
+        await sleep(params.pollMs ?? 250, params.signal);
     }
 }
 //# sourceMappingURL=recorder.js.map