@qearlyao/familiar 0.1.0

package/dist/memory/lcm/backfill.js

@@ -0,0 +1,247 @@
+import { readdir, readFile } from "node:fs/promises";
+import { relative, resolve } from "node:path";
+import { indexLcmRecords } from "./indexer.js";
+import { normalizeChatRecords } from "./normalize.js";
+import { computeLcmRecordKey } from "./store.js";
+const DEFAULT_YIELD_EVERY_N = 1024;
+const INDEX_BATCH_SIZE = 32;
+export async function backfillFromChatLogs(deps, options) {
+    void deps.memoryStore;
+    void deps.embeddingProvider;
+    const report = emptyReport();
+    const dataDir = resolve(options.dataDir);
+    const yieldEveryN = options.yieldEveryN ?? DEFAULT_YIELD_EVERY_N;
+    const channels = options.channels ? new Set(options.channels) : null;
+    let recordsProcessed = 0;
+    const emit = (phase, file) => {
+        options.onProgress?.({ phase, file, recordsProcessed, report: { ...report } });
+    };
+    const tick = async () => {
+        if (options.signal?.aborted)
+            return false;
+        if (yieldEveryN > 0 && recordsProcessed > 0 && recordsProcessed % yieldEveryN === 0) {
+            await new Promise((resolveYield) => setTimeout(resolveYield, 0));
+        }
+        return !options.signal?.aborted;
+    };
+    for (const chatFile of await listChatFiles(dataDir, channels, report)) {
+        if (options.signal?.aborted)
+            break;
+        emit("chat_file", chatFile.sourcePath);
+        let records;
+        try {
+            records = await readChatLogFile(chatFile.filePath);
+        }
+        catch (error) {
+            report.errors.push(formatError(`Failed to read chat log ${chatFile.sourcePath}`, error));
+            continue;
+        }
+        report.chatFilesProcessed += 1;
+        for (const group of groupByBackfillSegment(chatFile.channelKey, records)) {
+            if (options.signal?.aborted)
+                break;
+            recordsProcessed += group.records.length;
+            const batch = normalizeChatRecords(group.records, {
+                segmentId: group.segmentId,
+                sessionId: group.segmentId,
+                channelKey: chatFile.channelKey,
+                sourcePath: chatFile.sourcePath,
+            });
+            if (batch.segments.length === 0 && batch.records.length === 0) {
+                if (!(await tick()))
+                    break;
+                continue;
+            }
+            if (options.dryRun) {
+                report.segmentsCreated += countMissingSegments(deps.lcmStore, batch.segments.map((segment) => segment.id));
+                const existing = countExistingRecords(deps.lcmStore, batch.records);
+                report.recordsSkippedDuplicate += existing;
+                report.recordsInserted += batch.records.length - existing;
+            }
+            else {
+                for (const segment of batch.segments) {
+                    if (!deps.lcmStore.getSegment(segment.id))
+                        report.segmentsCreated += 1;
+                    deps.lcmStore.ensureSegment(segment);
+                }
+                const inserted = [];
+                for (const record of batch.records) {
+                    if (recordExists(deps.lcmStore, record)) {
+                        report.recordsSkippedDuplicate += 1;
+                        continue;
+                    }
+                    const id = deps.lcmStore.insertRecord(record);
+                    const stored = deps.lcmStore.getRecord(id);
+                    if (!stored)
+                        throw new Error(`Failed to read backfilled LCM record: ${id}`);
+                    inserted.push(stored);
+                    report.recordsInserted += 1;
+                    if (inserted.length >= INDEX_BATCH_SIZE) {
+                        report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
+                        inserted.length = 0;
+                    }
+                }
+                if (inserted.length > 0) {
+                    report.indexedChunks += (await indexLcmRecords({ indexer: deps.indexer, records: inserted, signal: options.signal })).ids.length;
+                }
+            }
+            if (!(await tick()))
+                break;
+            emit("chat_records", chatFile.sourcePath);
+        }
+    }
+    for (const transcriptPath of await listTranscriptFiles(dataDir, report)) {
+        if (options.signal?.aborted)
+            break;
+        report.transcriptFilesProcessed += 1;
+        emit("transcript_file", relative(dataDir, transcriptPath));
+        // TODO: Transcript JSONL rows do not currently have a Familiar-owned, typed
+        // schema that can be safely matched to ChatLogRecord by (jobId, timestamp).
+        // Keep transcript discovery here, but avoid force-fitting structured parts.
+        if (!(await tick()))
+            break;
+    }
+    emit(options.signal?.aborted ? "aborted" : "complete");
+    return report;
+}
+function emptyReport() {
+    return {
+        chatFilesProcessed: 0,
+        transcriptFilesProcessed: 0,
+        recordsInserted: 0,
+        recordsSkippedDuplicate: 0,
+        segmentsCreated: 0,
+        summariesInserted: 0,
+        indexedChunks: 0,
+        errors: [],
+    };
+}
+async function listChatFiles(dataDir, channels, report) {
+    const chatDir = resolve(dataDir, "chat");
+    let channelEntries;
+    try {
+        channelEntries = await readdir(chatDir, { withFileTypes: true });
+    }
+    catch (error) {
+        if (errorCode(error) === "ENOENT")
+            return [];
+        report.errors.push(formatError(`Failed to read chat directory ${chatDir}`, error));
+        return [];
+    }
+    const files = [];
+    for (const entry of channelEntries
+        .filter((item) => item.isDirectory())
+        .sort((a, b) => a.name.localeCompare(b.name))) {
+        if (channels && !channels.has(entry.name))
+            continue;
+        const channelDir = resolve(chatDir, entry.name);
+        let dateEntries;
+        try {
+            dateEntries = await readdir(channelDir, { withFileTypes: true });
+        }
+        catch (error) {
+            report.errors.push(formatError(`Failed to read channel directory ${channelDir}`, error));
+            continue;
+        }
+        for (const file of dateEntries.filter((item) => item.isFile() && /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(item.name))) {
+            const filePath = resolve(channelDir, file.name);
+            files.push({
+                channelKey: entry.name,
+                filePath,
+                sourcePath: relative(dataDir, filePath),
+            });
+        }
+    }
+    return files.sort((a, b) => a.sourcePath.localeCompare(b.sourcePath));
+}
+async function listTranscriptFiles(dataDir, report) {
+    const transcriptDir = resolve(dataDir, "transcripts");
+    let entries;
+    try {
+        entries = await readdir(transcriptDir, { withFileTypes: true });
+    }
+    catch (error) {
+        if (errorCode(error) === "ENOENT")
+            return [];
+        report.errors.push(formatError(`Failed to read transcript directory ${transcriptDir}`, error));
+        return [];
+    }
+    return entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
+        .map((entry) => resolve(transcriptDir, entry.name))
+        .sort();
+}
+async function readChatLogFile(filePath) {
+    const content = await readFile(filePath, "utf8");
+    const records = [];
+    for (const [index, line] of content.split(/\r?\n/).entries()) {
+        if (!line.trim())
+            continue;
+        const parsed = JSON.parse(line);
+        if (!isChatLogRecord(parsed))
+            throw new Error(`Malformed chat log record: ${filePath}:${index + 1}`);
+        records.push(parsed);
+    }
+    return records.sort((a, b) => a.recordId - b.recordId);
+}
+function isChatLogRecord(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const record = value;
+    return typeof record.recordId === "number" && typeof record.ts === "string" && typeof record.type === "string";
+}
+function groupByBackfillSegment(channelKey, records) {
+    const groups = [];
+    let current = [];
+    let sequence = 0;
+    const flush = () => {
+        const firstRecord = current[0];
+        if (!firstRecord)
+            return;
+        groups.push({
+            segmentId: `backfill-${channelKey}-${sanitizeSegmentIdPart(firstRecord.ts)}-${sequence}`,
+            records: current,
+        });
+        sequence += 1;
+        current = [];
+    };
+    for (const record of records) {
+        if (record.type === "control" && record.command === "new" && current.length > 0)
+            flush();
+        current.push(record);
+    }
+    flush();
+    return groups;
+}
+function sanitizeSegmentIdPart(value) {
+    return value.replace(/[^A-Za-z0-9._=-]+/g, "_").slice(0, 160) || "unknown";
+}
+function countMissingSegments(lcmStore, segmentIds) {
+    let missing = 0;
+    for (const segmentId of new Set(segmentIds)) {
+        if (!lcmStore.getSegment(segmentId))
+            missing += 1;
+    }
+    return missing;
+}
+function countExistingRecords(lcmStore, records) {
+    let existing = 0;
+    for (const record of records) {
+        if (recordExists(lcmStore, record))
+            existing += 1;
+    }
+    return existing;
+}
+function recordExists(lcmStore, record) {
+    return !!lcmStore.db
+        .prepare("SELECT 1 FROM lcm_records WHERE record_key = ? LIMIT 1")
+        .get(computeLcmRecordKey(record));
+}
+function errorCode(error) {
+    return error && typeof error === "object" && "code" in error
+        ? String(error.code)
+        : undefined;
+}
+function formatError(prefix, error) {
+    return `${prefix}: ${error instanceof Error ? error.message : String(error)}`;
+}

package/dist/memory/lcm/condense.js

@@ -0,0 +1,146 @@
+import { estimateTextTokens, selectRetainedSummaries } from "./context.js";
+import { indexLcmSummaries } from "./indexer.js";
+import { capSummaryText } from "./summarizer.js";
+export async function condense(input) {
+    const groupSize = positiveInteger(input.config.condenseGroupSize, "condenseGroupSize");
+    const maxDepth = positiveInteger(input.config.maxSummaryDepth, "maxSummaryDepth");
+    if (input.depth >= maxDepth)
+        return [];
+    const children = input.store
+        .listSummaries(input.segmentId)
+        .filter((summary) => summary.depth === input.depth &&
+        summary.status === "ready" &&
+        (input.candidateIds === undefined || input.candidateIds.includes(summary.id)) &&
+        input.store.getSummaryChildren(summary.id).length === 0)
+        .sort(compareCoverage);
+    const created = [];
+    for (let index = 0; index + groupSize <= children.length; index += groupSize) {
+        const group = children.slice(index, index + groupSize);
+        const coversFromRecordId = minNullable(group.map((summary) => summary.coversFromRecordId));
+        const coversToRecordId = maxNullable(group.map((summary) => summary.coversToRecordId));
+        const text = capSummaryText(await summarizeCondensedGroup({
+            group,
+            targetTokens: input.config.leafTargetTokens,
+            depth: input.depth + 1,
+            summarizer: input.summarizer,
+            signal: input.signal,
+        }), input.config.leafTargetTokens);
+        const id = input.store.insertSummary({
+            segmentId: input.segmentId,
+            depth: input.depth + 1,
+            status: "ready",
+            text,
+            coversFromRecordId,
+            coversToRecordId,
+            source: {
+                sourceType: "manual",
+                sourceRef: `lcm_condense:d${input.depth + 1}:${group.map((summary) => summary.id).join("-")}`,
+            },
+            sourceItems: group.map((summary) => ({ summaryId: summary.id, sourceRef: `lcm_summary:${summary.id}` })),
+            parents: group.map((summary) => summary.id),
+            metadata: {
+                source: "condense",
+                childDepth: input.depth,
+                childSummaryIds: group.map((summary) => summary.id),
+                ...coverageMetadataFromSummaries(group),
+            },
+        });
+        const summary = input.store.getSummary(id);
+        if (summary) {
+            created.push(summary);
+            if (input.indexer) {
+                await indexLcmSummaries({ indexer: input.indexer, summaries: [summary], signal: input.signal }).catch((error) => console.error("memory LCM condensed summary indexing failed", error));
+            }
+        }
+    }
+    if (created.length > 0 && input.depth + 1 < maxDepth) {
+        created.push(...(await condense({
+            ...input,
+            depth: input.depth + 1,
+            candidateIds: created.map((summary) => summary.id),
+        })));
+    }
+    return created;
+}
+export function renderCondensedSummariesForContext(summaries) {
+    return selectRetainedSummaries(summaries);
+}
+function compareCoverage(a, b) {
+    return ((a.coversFromRecordId ?? Number.MAX_SAFE_INTEGER) - (b.coversFromRecordId ?? Number.MAX_SAFE_INTEGER) ||
+        (a.coversToRecordId ?? Number.MAX_SAFE_INTEGER) - (b.coversToRecordId ?? Number.MAX_SAFE_INTEGER) ||
+        a.id - b.id);
+}
+async function summarizeCondensedGroup(input) {
+    const text = renderCondensedSummaryInput(input.group);
+    if (input.summarizer.summarizeCondensed) {
+        return input.summarizer.summarizeCondensed({
+            text,
+            targetTokens: input.targetTokens,
+            depth: input.depth,
+            childSummaryCount: input.group.length,
+        }, input.signal);
+    }
+    return input.summarizer.summarizeLeaf({
+        text,
+        targetTokens: input.targetTokens,
+        mode: "normal",
+    }, input.signal);
+}
+function renderCondensedSummaryInput(group) {
+    return group
+        .map((summary) => [
+        `<summary id="${summary.id}" depth="${summary.depth}" from="${summary.coversFromRecordId ?? ""}" to="${summary.coversToRecordId ?? ""}" tokens="${estimateTextTokens(summary.text)}">`,
+        formatSummaryTimeRange(summary),
+        summary.text,
+        "</summary>",
+    ]
+        .filter(Boolean)
+        .join("\n"))
+        .join("\n\n");
+}
+function formatSummaryTimeRange(summary) {
+    const from = metadataString(summary.metadata?.coverageFromHappenedAt ?? summary.metadata?.timestamp);
+    const to = metadataString(summary.metadata?.coverageToHappenedAt ?? summary.metadata?.timestamp);
+    if (!from && !to)
+        return "";
+    return `[time_range ${from ?? "unknown"} - ${to ?? "unknown"}]`;
+}
+function metadataString(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+function minNullable(values) {
+    const present = values.filter((value) => value !== null);
+    return present.length === 0 ? null : Math.min(...present);
+}
+function maxNullable(values) {
+    const present = values.filter((value) => value !== null);
+    return present.length === 0 ? null : Math.max(...present);
+}
+function coverageMetadataFromSummaries(group) {
+    const from = firstValidTime(group.map((summary) => summary.metadata?.coverageFromHappenedAt ?? summary.metadata?.timestamp));
+    const to = lastValidTime(group.map((summary) => summary.metadata?.coverageToHappenedAt ?? summary.metadata?.timestamp));
+    return {
+        ...(from ? { coverageFromHappenedAt: from } : {}),
+        ...(to ? { coverageToHappenedAt: to, timestamp: to } : {}),
+    };
+}
+function firstValidTime(values) {
+    for (const value of values) {
+        if (typeof value === "string" && Number.isFinite(Date.parse(value)))
+            return value;
+    }
+    return null;
+}
+function lastValidTime(values) {
+    for (let index = values.length - 1; index >= 0; index -= 1) {
+        const value = values[index];
+        if (typeof value === "string" && Number.isFinite(Date.parse(value)))
+            return value;
+    }
+    return null;
+}
+function positiveInteger(value, name) {
+    if (!Number.isInteger(value) || value < 1)
+        throw new Error(`${name} must be an integer >= 1`);
+    return value;
+}