@memorax/memorax-code 0.1.0

package/lib/memorax-code-backend/dist/memory-writeback-buffer.js

@@ -0,0 +1,217 @@
+import { writebackMessagesContentChars } from "./memory-writeback-chunk.js";
+import { memoryWritebackBufferConfig } from "./memorax-config.js";
+const SYSTEM_CLOCK = {
+    now: () => Date.now(),
+    setTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+    clearTimeout: (timer) => clearTimeout(timer),
+};
+export function createMemoryWritebackBufferRuntime() {
+    const writebackBuffers = new Map();
+    return {
+        enqueue(decision, options, deps) {
+            enqueueMemoryWritebackBufferForRuntime(writebackBuffers, decision, options, deps);
+        },
+        flushAll(flushReason) {
+            let flushed = 0;
+            for (const [bufferKey, buffer] of [...writebackBuffers.entries()]) {
+                if (flushMemoryWritebackBuffer(writebackBuffers, bufferKey, flushReason, buffer.deps)) {
+                    flushed += 1;
+                }
+            }
+            return flushed;
+        },
+        close() {
+            for (const buffer of writebackBuffers.values()) {
+                if (buffer.timer)
+                    buffer.clock.clearTimeout(buffer.timer);
+            }
+            writebackBuffers.clear();
+        },
+    };
+}
+function enqueueMemoryWritebackBufferForRuntime(writebackBuffers, decision, options, deps) {
+    const env = options.env ?? process.env;
+    const config = memoryWritebackBufferConfig(env);
+    if (!options.repositoryScope)
+        return;
+    const bufferKey = memoryWritebackBufferKey(decision.client, decision.sessionKey, options.repositoryScope);
+    let buffer = writebackBuffers.get(bufferKey);
+    if (buffer?.turnKeys.has(decision.idempotencyKey)) {
+        deps.debug("memory.automatic_writeback", {
+            scheduled: false,
+            skipReason: "buffer_duplicate_turn",
+            sessionKey: decision.sessionKey,
+        });
+        return;
+    }
+    const turnContentChars = writebackMessagesContentChars(decision.messages);
+    if (buffer && buffer.contentChars + turnContentChars > config.maxChars) {
+        flushMemoryWritebackBuffer(writebackBuffers, bufferKey, "char_limit", deps);
+        buffer = undefined;
+    }
+    if (!buffer) {
+        buffer = createMemoryWritebackBuffer(bufferKey, decision.client, decision.sessionKey, env, options, options.repositoryScope, deps.clock ?? SYSTEM_CLOCK, deps);
+        writebackBuffers.set(bufferKey, buffer);
+    }
+    buffer.turns.push({
+        idempotencyKey: decision.idempotencyKey,
+        messages: decision.messages,
+        ...(options.traceContext ? { traceContext: options.traceContext } : {}),
+    });
+    buffer.turnKeys.add(decision.idempotencyKey);
+    buffer.contentChars += turnContentChars;
+    const clock = deps.clock ?? SYSTEM_CLOCK;
+    buffer.updatedAt = clock.now();
+    buffer.env = env;
+    buffer.fetchImpl = options.fetchImpl;
+    buffer.memoryObservability = options.memoryObservability;
+    buffer.memoryObservabilitySource = options.memoryObservabilitySource;
+    buffer.deps = deps;
+    deps.debug("memory.automatic_writeback", {
+        scheduled: false,
+        buffered: true,
+        sessionKey: decision.sessionKey,
+        turnCount: buffer.turns.length,
+        messageCount: bufferedMessages(buffer).length,
+        contentChars: buffer.contentChars,
+    });
+    if (buffer.turns.length >= config.maxTurns) {
+        flushMemoryWritebackBuffer(writebackBuffers, bufferKey, "turn_limit", deps);
+    }
+    else if (buffer.contentChars > config.maxChars) {
+        flushMemoryWritebackBuffer(writebackBuffers, bufferKey, "char_limit", deps);
+    }
+    else {
+        resetMemoryWritebackIdleTimer(writebackBuffers, buffer, config.maxAgeMs, deps);
+    }
+}
+function createMemoryWritebackBuffer(bufferKey, client, sessionKey, env, options, repositoryScope, clock, deps) {
+    const now = clock.now();
+    const buffer = {
+        bufferKey,
+        client,
+        sessionKey,
+        turns: [],
+        turnKeys: new Set(),
+        contentChars: 0,
+        createdAt: now,
+        updatedAt: now,
+        timerGeneration: 0,
+        clock,
+        env,
+        fetchImpl: options.fetchImpl,
+        memoryObservability: options.memoryObservability,
+        memoryObservabilitySource: options.memoryObservabilitySource,
+        repositoryScope,
+        deps,
+    };
+    return buffer;
+}
+function resetMemoryWritebackIdleTimer(writebackBuffers, buffer, maxAgeMs, deps) {
+    if (buffer.timer)
+        buffer.clock.clearTimeout(buffer.timer);
+    const clock = deps.clock ?? SYSTEM_CLOCK;
+    const generation = buffer.timerGeneration + 1;
+    buffer.clock = clock;
+    buffer.timerGeneration = generation;
+    buffer.idleDeadlineAt = buffer.updatedAt + maxAgeMs;
+    const delayMs = Math.max(0, buffer.idleDeadlineAt - clock.now());
+    buffer.timer = clock.setTimeout(() => {
+        const current = writebackBuffers.get(buffer.bufferKey);
+        if (current !== buffer || current.timerGeneration !== generation)
+            return;
+        flushMemoryWritebackBuffer(writebackBuffers, buffer.bufferKey, "idle_limit", deps);
+    }, delayMs);
+    if (typeof buffer.timer.unref === "function")
+        buffer.timer.unref();
+}
+function flushMemoryWritebackBuffer(writebackBuffers, bufferKey, flushReason, deps) {
+    const buffer = writebackBuffers.get(bufferKey);
+    if (!buffer)
+        return false;
+    writebackBuffers.delete(bufferKey);
+    if (buffer.timer)
+        buffer.clock.clearTimeout(buffer.timer);
+    if (buffer.turns.length === 0)
+        return false;
+    const sessionKey = buffer.sessionKey;
+    const messages = bufferedMessages(buffer);
+    const scopeHash = deps.hashText(buffer.repositoryScope.effectiveUserId);
+    const idempotencyKey = `automatic-buffer:v1:${buffer.client}:${scopeHash}:${sessionKey}:${deps.hashText(messages.map((message) => `${message.role}:${message.content}`).join("\n"))}`;
+    const dedupeKeys = [idempotencyKey, ...buffer.turns.map((turn) => turn.idempotencyKey)];
+    if (dedupeKeys.some((key) => deps.hasPendingWriteback(key)))
+        return false;
+    deps.reservePendingWritebacks(dedupeKeys);
+    deps.debug("memory.automatic_writeback", {
+        scheduled: true,
+        buffered: true,
+        sessionKey,
+        idempotencyKey,
+        flushReason,
+        turnCount: buffer.turns.length,
+        messageCount: messages.length,
+        contentChars: buffer.contentChars,
+    });
+    deps.flush({
+        client: buffer.client,
+        sessionKey,
+        idempotencyKey,
+        messages,
+        dedupeKeys,
+        flushReason,
+        turnCount: buffer.turns.length,
+    }, {
+        sessionKey,
+        env: buffer.env,
+        fetchImpl: buffer.fetchImpl,
+        memoryObservability: buffer.memoryObservability,
+        memoryObservabilitySource: buffer.memoryObservabilitySource,
+        repositoryScope: buffer.repositoryScope,
+        relatedTurns: relatedTurnsForBuffer(buffer),
+        traceContext: sessionTraceContextForBuffer(buffer),
+    });
+    return true;
+}
+function memoryWritebackBufferKey(client, sessionKey, repositoryScope) {
+    return JSON.stringify([
+        client,
+        repositoryScope.effectiveUserId,
+        sessionKey,
+    ]);
+}
+function bufferedMessages(buffer) {
+    return buffer.turns.flatMap((turn) => turn.messages);
+}
+function relatedTurnsForBuffer(buffer) {
+    const relatedTurns = [];
+    for (const turn of buffer.turns) {
+        const context = turn.traceContext;
+        if (!context)
+            continue;
+        relatedTurns.push({
+            ...(context.turnId ? { turnId: context.turnId } : {}),
+            ...(context.requestId ? { requestId: context.requestId } : {}),
+            ...(context.nativeRequestId ? { nativeRequestId: context.nativeRequestId } : {}),
+            contextOrigin: context.contextOrigin,
+            capturedAt: context.capturedAt,
+        });
+    }
+    return relatedTurns.length > 0 ? relatedTurns : undefined;
+}
+function sessionTraceContextForBuffer(buffer) {
+    const contexts = buffer.turns.flatMap((turn) => turn.traceContext ? [turn.traceContext] : []);
+    const context = contexts.find((candidate) => candidate.sessionId === buffer.sessionKey) ?? contexts[0];
+    if (!context)
+        return undefined;
+    return {
+        schemaVersion: context.schemaVersion,
+        client: context.client,
+        sessionId: context.sessionId,
+        ...(context.threadId ? { threadId: context.threadId } : {}),
+        ...(context.transcriptPath ? { transcriptPath: context.transcriptPath } : {}),
+        ...(context.cwd ? { cwd: context.cwd } : {}),
+        ...(context.workspaceKind ? { workspaceKind: context.workspaceKind } : {}),
+        contextOrigin: context.contextOrigin,
+        capturedAt: context.capturedAt,
+    };
+}

package/lib/memorax-code-backend/dist/memory-writeback-chunk.js

@@ -0,0 +1,120 @@
+import { createHash } from "node:crypto";
+import { memoryWritebackChunkConfig, memoryWritebackChunkEnabled, } from "./memorax-config.js";
+const CODE_CHUNK_GROUP_ID_PREFIX = "memory-writeback-chunk:v1:";
+export function memoryWritebackAddParts(decision, env) {
+    if (!memoryWritebackChunkEnabled(env)) {
+        return [{ idempotencyKey: decision.idempotencyKey, messages: decision.messages }];
+    }
+    const config = memoryWritebackChunkConfig(env);
+    const payloads = chunkWritebackMessages(decision.messages, config);
+    if (payloads.length <= 1) {
+        return [{ idempotencyKey: decision.idempotencyKey, messages: payloads[0] ?? decision.messages }];
+    }
+    const groupId = codeChunkGroupId(decision.idempotencyKey);
+    return payloads.map((messages, index) => ({
+        idempotencyKey: `${decision.idempotencyKey}:part:${index}`,
+        messages,
+        chunk: {
+            group_id: groupId,
+            index,
+            count: payloads.length,
+        },
+    }));
+}
+function codeChunkGroupId(idempotencyKey) {
+    return `${CODE_CHUNK_GROUP_ID_PREFIX}${createHash("sha256").update(idempotencyKey).digest("hex")}`;
+}
+export function writebackMessagesContentChars(messages) {
+    return messages.reduce((total, message) => total + message.content.length, 0);
+}
+function chunkWritebackMessages(messages, config) {
+    const turns = writebackMessageTurns(messages);
+    const payloads = [];
+    let current = [];
+    const flushCurrent = () => {
+        if (current.length > 0) {
+            payloads.push(current);
+            current = [];
+        }
+    };
+    const appendCurrent = (payload) => {
+        if (payload.length === 0)
+            return;
+        if (current.length > 0 && writebackMessagesContentChars([...current, ...payload]) > config.maxChars) {
+            flushCurrent();
+        }
+        current.push(...payload);
+    };
+    for (const turn of turns) {
+        const userChunks = turn.user ? chunkWritebackMessage(turn.user, config) : [];
+        const assistantChunks = turn.assistants.flatMap((message) => chunkWritebackMessage(message, config));
+        const needsDedicatedPayloads = userChunks.length > 1 || assistantChunks.length > turn.assistants.length;
+        if (!needsDedicatedPayloads) {
+            appendCurrent([...userChunks, ...assistantChunks]);
+            continue;
+        }
+        flushCurrent();
+        if (userChunks.length > 1) {
+            if (assistantChunks.length === 0) {
+                for (const userChunk of userChunks)
+                    payloads.push([userChunk]);
+                continue;
+            }
+            for (const userChunk of userChunks.slice(0, -1))
+                payloads.push([userChunk]);
+            payloads.push([userChunks[userChunks.length - 1], assistantChunks[0]]);
+            for (const assistantChunk of assistantChunks.slice(1))
+                payloads.push([assistantChunk]);
+            continue;
+        }
+        if (assistantChunks.length === 0) {
+            if (userChunks.length === 1)
+                payloads.push(userChunks);
+            continue;
+        }
+        if (userChunks.length === 1) {
+            payloads.push([userChunks[0], assistantChunks[0]]);
+            for (const assistantChunk of assistantChunks.slice(1))
+                payloads.push([assistantChunk]);
+            continue;
+        }
+        for (const assistantChunk of assistantChunks)
+            payloads.push([assistantChunk]);
+    }
+    flushCurrent();
+    return payloads.filter((payload) => payload.length > 0);
+}
+function writebackMessageTurns(messages) {
+    const turns = [];
+    for (const message of messages) {
+        if (message.role === "user") {
+            turns.push({ user: message, assistants: [] });
+            continue;
+        }
+        const turn = turns[turns.length - 1] ?? { assistants: [] };
+        if (turns.length === 0)
+            turns.push(turn);
+        turn.assistants.push(message);
+    }
+    return turns;
+}
+function chunkWritebackMessage(message, config) {
+    return splitTextWithOverlap(message.content, config.maxChars, config.overlapRatio)
+        .map((content) => ({ role: message.role, content }))
+        .filter((part) => part.content.trim());
+}
+function splitTextWithOverlap(text, maxChars, overlapRatio) {
+    if (text.length <= maxChars)
+        return [text];
+    const overlap = Math.min(Math.floor(maxChars * overlapRatio), Math.floor(maxChars / 2));
+    const step = Math.max(1, maxChars - overlap);
+    const chunks = [];
+    for (let start = 0; start < text.length;) {
+        const end = Math.min(text.length, start + maxChars);
+        chunks.push(text.slice(start, end));
+        if (end >= text.length)
+            break;
+        start += step;
+    }
+    return chunks;
+}

package/lib/memorax-code-backend/dist/memory-writeback-reconciler.js

@@ -0,0 +1,244 @@
+import { createHash } from "node:crypto";
+import { backendDebug } from "./debug-log.js";
+import { callMemoAddStatus, memoraxConfigFromEnv } from "./memorax-adapter.js";
+import { MEMORAX_PROVIDER_ID, memoraxWritebackEnabled } from "./memorax-config.js";
+import { completeMemoryWritebackTask, createMemoryWritebackTaskProjection, } from "./memory-writeback-task-projection.js";
+import { clientTraceConfigFromEnv } from "./trace-config.js";
+import { recordTraceEvent } from "./trace-store.js";
+const DEFAULT_INTERVAL_MS = 60_000;
+const DEFAULT_MAX_TASKS_PER_RUN = 20;
+const DEFAULT_CONCURRENCY = 4;
+const DEFAULT_MAX_BACKOFF_MS = 60 * 60_000;
+export async function reconcileMemoryWritebackStatuses(options) {
+    return reconcileMemoryWritebackStatusesWithPolicy(options);
+}
+export function startMemoryWritebackReconciler(options) {
+    let closed = false;
+    let active;
+    let candidateOffset = 0;
+    const intervalMs = positiveInteger(options.intervalMs, DEFAULT_INTERVAL_MS);
+    const maxTasks = positiveInteger(options.maxTasks, DEFAULT_MAX_TASKS_PER_RUN);
+    const taskProjection = resolveTaskProjection(options);
+    const policy = createReconcileBackoffPolicy({
+        now: options.now ?? Date.now,
+        baseDelayMs: intervalMs,
+        maxDelayMs: Math.max(intervalMs, positiveInteger(options.maxBackoffMs, DEFAULT_MAX_BACKOFF_MS)),
+    });
+    const runNow = () => {
+        if (closed)
+            return Promise.resolve(emptyReport());
+        if (active)
+            return active;
+        active = reconcileMemoryWritebackStatusesWithPolicy({
+            ...options,
+            maxTasks,
+            candidateOffset,
+            taskProjection,
+        }, policy).finally(() => {
+            candidateOffset += maxTasks;
+            active = undefined;
+        });
+        return active;
+    };
+    const scheduleRun = () => {
+        void runNow().catch((error) => {
+            backendDebug("memory_writeback_reconciler.run_failed", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        });
+    };
+    scheduleRun();
+    const timer = setInterval(scheduleRun, intervalMs);
+    timer.unref?.();
+    return {
+        runNow,
+        async close() {
+            closed = true;
+            clearInterval(timer);
+            await active?.catch(() => undefined);
+        },
+    };
+}
+async function reconcileMemoryWritebackStatusesWithPolicy(options, policy) {
+    const env = { ...(options.env ?? process.env), MEMORAX_CODE_HOME: options.memoraxCodeHome };
+    const config = memoraxConfigFromEnv(env);
+    const taskProjection = resolveTaskProjection(options);
+    if (!clientTraceConfigFromEnv(taskProjection.client, env).enabled
+        || !config.ok
+        || !memoraxWritebackEnabled(env)) {
+        return emptyReport();
+    }
+    const maxTasks = positiveInteger(options.maxTasks, DEFAULT_MAX_TASKS_PER_RUN);
+    const concurrency = positiveInteger(options.concurrency, DEFAULT_CONCURRENCY);
+    const pendingWritebacks = await taskProjection.listPending();
+    policy?.retainPending(pendingWritebacks);
+    const dueWritebacks = pendingWritebacks.filter((task) => policy?.isDue(task) ?? true);
+    const candidates = rotatedBatch(dueWritebacks, maxTasks, options.candidateOffset ?? 0);
+    const queue = [...candidates];
+    const report = { inspected: 0, persisted: 0, pending: 0, failed: 0 };
+    const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+        for (;;) {
+            const task = queue.shift();
+            if (!task)
+                return;
+            report.inspected += 1;
+            try {
+                const raw = await callMemoAddStatus(config.config, task.taskId, options.fetchImpl);
+                const status = normalizeAddStatus(raw);
+                if (isPendingWritebackStatus(status.status)) {
+                    report.pending += 1;
+                    policy?.recordResult(task, "pending");
+                    continue;
+                }
+                await persistTerminalWritebackStatus(options.memoraxCodeHome, env, task, status);
+                report.persisted += 1;
+                policy?.recordResult(task, "persisted");
+            }
+            catch (error) {
+                report.failed += 1;
+                policy?.recordResult(task, "failed");
+                backendDebug("memory_writeback_reconciler.status_failed", {
+                    taskId: task.taskId,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    });
+    await Promise.all(workers);
+    return report;
+}
+function resolveTaskProjection(options) {
+    return options.taskProjection ?? createMemoryWritebackTaskProjection({
+        memoraxCodeHome: options.memoraxCodeHome,
+        client: options.client ?? "codex",
+    });
+}
+async function persistTerminalWritebackStatus(memoraxCodeHome, env, task, status) {
+    const completed = completeMemoryWritebackTask({
+        status: status.status,
+        ...(status.memoryKnown ? { memory: status.memory, memoryKnown: true } : {}),
+        error: status.error,
+    });
+    const capturedAt = new Date().toISOString();
+    const recorded = await recordTraceEvent({
+        memoraxCodeHome,
+        env,
+        eventId: terminalEventId(task, completed.status),
+        traceContext: {
+            schemaVersion: "1",
+            client: task.client,
+            sessionId: task.sessionId,
+            ...(task.turnId ? { turnId: task.turnId } : {}),
+            ...(task.memoryProject ? { memoryProject: task.memoryProject } : {}),
+            contextOrigin: "manual",
+            capturedAt,
+        },
+        type: "memory_writeback_status",
+        source: "writeback_reconciler",
+        operation: "writeback",
+        ok: completed.ok,
+        request: {
+            task_id: task.taskId,
+            original_event_id: task.eventId,
+            provider: MEMORAX_PROVIDER_ID,
+        },
+        response: {
+            taskId: task.taskId,
+            status: completed.status,
+            outcome: completed.outcome,
+            ...(completed.savedMemoryCount === undefined ? {} : { savedMemoryCount: completed.savedMemoryCount }),
+            ...(completed.savedMemoryIds?.length ? { savedMemoryIds: completed.savedMemoryIds } : {}),
+            ...(completed.savedMemories?.length ? { savedMemories: completed.savedMemories } : {}),
+        },
+        error: completed.error,
+    });
+    if (!recorded.written && recorded.reason !== "duplicate_event") {
+        throw new Error(`writeback terminal status was not persisted: ${recorded.reason}`);
+    }
+}
+function createReconcileBackoffPolicy(options) {
+    const retries = new Map();
+    return {
+        retainPending(tasks) {
+            const pendingKeys = new Set(tasks.map(reconcileCandidateKey));
+            for (const key of retries.keys()) {
+                if (!pendingKeys.has(key))
+                    retries.delete(key);
+            }
+        },
+        isDue(task) {
+            const key = reconcileCandidateKey(task);
+            return (retries.get(key)?.nextAttemptAt ?? 0) <= options.now();
+        },
+        recordResult(task, result) {
+            const key = reconcileCandidateKey(task);
+            if (result === "persisted") {
+                retries.delete(key);
+                return;
+            }
+            const attempts = (retries.get(key)?.attempts ?? 0) + 1;
+            const multiplier = 2 ** Math.min(attempts - 1, 30);
+            const delayMs = Math.min(options.maxDelayMs, options.baseDelayMs * multiplier);
+            retries.set(key, {
+                attempts,
+                nextAttemptAt: options.now() + delayMs,
+            });
+        },
+    };
+}
+function reconcileCandidateKey(task) {
+    return JSON.stringify([
+        task.client,
+        task.sessionId,
+        task.eventId,
+        task.taskId,
+    ]);
+}
+function normalizeAddStatus(raw) {
+    const envelope = record(raw);
+    const data = record(envelope?.data) ?? envelope;
+    const status = typeof data?.status === "string" ? data.status.trim().toLowerCase() : "unknown";
+    const error = typeof data?.error === "string" && data.error.trim() ? data.error.trim() : null;
+    const hasMemoryField = Boolean(data)
+        && (Object.prototype.hasOwnProperty.call(data, "memory")
+            || Object.prototype.hasOwnProperty.call(data, "memories"));
+    const memory = data?.memory ?? data?.memories;
+    const memoryKnown = hasMemoryField && memory !== null && memory !== undefined;
+    return {
+        status,
+        ...(memoryKnown ? { memory } : {}),
+        memoryKnown,
+        error,
+    };
+}
+function isPendingWritebackStatus(status) {
+    return status === "accepted"
+        || status === "processing"
+        || status === "queued"
+        || status === "pending"
+        || status === "unknown";
+}
+function terminalEventId(task, status) {
+    const digest = createHash("sha256")
+        .update(`${task.sessionId}\u0000${task.eventId}\u0000${task.taskId}\u0000${status}`)
+        .digest("hex")
+        .slice(0, 32);
+    return `memory-writeback-status-${digest}`;
+}
+function positiveInteger(value, fallback) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : fallback;
+}
+function rotatedBatch(items, limit, offset) {
+    if (items.length <= limit)
+        return [...items];
+    const start = Number.isSafeInteger(offset) && offset > 0 ? offset % items.length : 0;
+    return Array.from({ length: limit }, (_, index) => items[(start + index) % items.length]);
+}
+function emptyReport() {
+    return { inspected: 0, persisted: 0, pending: 0, failed: 0 };
+}
+function record(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : undefined;
+}