@qearlyao/familiar 0.1.0

package/dist/memory/lcm/normalize.js

@@ -0,0 +1,235 @@
+export function normalizeChatRecords(chatRecords, options) {
+    const segments = new Map();
+    const records = [];
+    const sortedRecords = [...chatRecords].sort((a, b) => a.recordId - b.recordId);
+    const assistantEventsByJob = collectAssistantEvents(sortedRecords);
+    for (const record of sortedRecords) {
+        const segmentId = options.segmentId;
+        if (!segments.has(segmentId)) {
+            segments.set(segmentId, {
+                id: segmentId,
+                sessionId: options.sessionId ?? null,
+                channelKey: options.channelKey ?? null,
+                startedAt: record.ts,
+            });
+        }
+        const source = chatRecordSource(record, options.sourcePath);
+        const common = {
+            segmentId,
+            happenedAt: record.ts,
+            sessionId: options.sessionId ?? null,
+            channelKey: options.channelKey ?? null,
+            channelId: record.channelId,
+            source,
+        };
+        if (record.type === "inbound") {
+            const text = withAttachmentNotes(record.text, attachmentNotes(record.attachments));
+            if (text) {
+                records.push({
+                    ...common,
+                    kind: "user",
+                    text,
+                    attachments: attachmentNotes(record.attachments),
+                    metadata: { authorId: record.authorId, authorName: record.authorName ?? null },
+                });
+            }
+            continue;
+        }
+        if (record.type === "outbound" && !record.silent) {
+            const notes = attachmentNotes(record.attachments ?? []);
+            const parts = assistantParts(record, assistantEventsByJob.get(record.jobId ?? ""));
+            appendAssistantFinalText(parts, withAttachmentNotes(record.text, notes));
+            const text = flattenLcmRecordParts(parts);
+            if (text) {
+                records.push({
+                    ...common,
+                    kind: "assistant",
+                    text,
+                    parts: parts.length ? parts : undefined,
+                    jobId: record.jobId ?? null,
+                    attachments: notes,
+                    metadata: { messageIds: record.messageIds, replyToMessageId: record.replyToMessageId ?? null },
+                });
+            }
+            continue;
+        }
+        if (record.type === "agent_event" && record.event.type === "tool_execution_end") {
+            const parts = [
+                {
+                    kind: "tool_result",
+                    toolCallId: record.event.toolCallId,
+                    toolName: record.event.toolName,
+                    output: record.event.result,
+                    ...(record.event.isError ? { isError: true } : {}),
+                },
+            ];
+            records.push({
+                ...common,
+                kind: "tool",
+                text: flattenLcmRecordParts(parts),
+                parts,
+                jobId: record.jobId,
+                metadata: { messageId: record.messageId },
+            });
+            continue;
+        }
+        if (record.type === "control" && record.command === "new") {
+            records.push({
+                ...common,
+                kind: "boundary",
+                text: record.text || "/new",
+                metadata: { command: record.command, args: record.args ?? null, authorId: record.authorId },
+            });
+            continue;
+        }
+        if (record.type === "runtime" && record.event === "reset") {
+            records.push({
+                ...common,
+                kind: "boundary",
+                text: record.detail || "runtime reset",
+                metadata: { event: record.event },
+            });
+        }
+    }
+    return { segments: [...segments.values()], records };
+}
+export function flattenLcmRecordParts(parts) {
+    return parts
+        .map((part) => {
+        if (part.kind === "text")
+            return part.text.trim();
+        if (part.kind === "thinking")
+            return part.text.trim() ? `[thinking] ${part.text.trim()}` : "";
+        if (part.kind === "tool_call")
+            return `[tool_call: ${part.toolName}(${briefJson(part.arguments)})]`;
+        return `[tool_result: ${part.toolName} -> ${briefJson(part.output)}]`;
+    })
+        .filter(Boolean)
+        .join("\n");
+}
+function collectAssistantEvents(records) {
+    const byJob = new Map();
+    for (const record of records) {
+        if (record.type !== "agent_event")
+            continue;
+        const events = byJob.get(record.jobId) ?? [];
+        events.push(record.event);
+        byJob.set(record.jobId, events);
+    }
+    return byJob;
+}
+function assistantParts(record, events) {
+    const parts = [];
+    if (record.thinking?.trim())
+        appendThinkingPart(parts, record.thinking);
+    for (const event of events ?? []) {
+        if (event.type !== "message_update")
+            continue;
+        const assistantEvent = event.assistantMessageEvent;
+        if (assistantEvent.type === "text_delta") {
+            appendTextPart(parts, assistantEvent.delta);
+        }
+        else if (assistantEvent.type === "thinking_delta") {
+            appendThinkingPart(parts, assistantEvent.delta);
+        }
+        else if (assistantEvent.type === "toolcall_end") {
+            parts.push({
+                kind: "tool_call",
+                toolCallId: assistantEvent.toolCall.id,
+                toolName: assistantEvent.toolCall.name,
+                arguments: assistantEvent.toolCall.arguments,
+            });
+        }
+    }
+    return parts;
+}
+function appendTextPart(parts, text) {
+    if (!text)
+        return;
+    const previous = parts.at(-1);
+    if (previous?.kind === "text") {
+        previous.text += text;
+        return;
+    }
+    parts.push({ kind: "text", text });
+}
+function appendAssistantFinalText(parts, text) {
+    const normalized = text.trim();
+    if (!normalized)
+        return;
+    const existingText = parts
+        .filter((part) => part.kind === "text")
+        .map((part) => part.text)
+        .join("")
+        .trim();
+    if (existingText === normalized)
+        return;
+    appendTextPart(parts, text);
+}
+function appendThinkingPart(parts, text) {
+    if (!text)
+        return;
+    const previous = parts.at(-1);
+    if (previous?.kind === "thinking" && !previous.signature) {
+        previous.text += text;
+        return;
+    }
+    parts.push({ kind: "thinking", text });
+}
+function briefJson(value, maxLength = 160) {
+    let text;
+    if (typeof value === "string")
+        text = value;
+    else {
+        try {
+            text = JSON.stringify(value);
+        }
+        catch {
+            text = String(value);
+        }
+    }
+    const compact = text.replace(/\s+/g, " ").trim();
+    return compact.length > maxLength ? `${compact.slice(0, maxLength - 1)}...` : compact;
+}
+function chatRecordSource(record, sourcePath) {
+    const messageId = "messageId" in record && typeof record.messageId === "string"
+        ? record.messageId
+        : "messageIds" in record
+            ? record.messageIds[0]
+            : undefined;
+    return {
+        sourceType: "chat",
+        sourcePath: sourcePath ?? null,
+        sourceRecordId: record.recordId,
+        sourceMessageId: messageId ?? null,
+        sourceRef: sourcePath ? `${sourcePath}#${record.recordId}` : `chat:${record.recordId}`,
+    };
+}
+function attachmentNotes(attachments) {
+    const notes = attachments
+        .map((attachment) => {
+        const derivedText = attachment.derived?.text?.text.trim();
+        const imageNote = attachment.derived?.image?.note?.trim();
+        const note = {
+            id: attachment.id,
+            name: attachment.name,
+            kind: attachment.kind,
+            mimeType: attachment.mimeType,
+            text: derivedText || undefined,
+            note: imageNote || undefined,
+            sourceRef: attachment.localPath ?? attachment.remoteUrl ?? attachment.sourceUrl,
+        };
+        return note.text || note.note || note.name ? note : null;
+    })
+        .filter((item) => item !== null);
+    return notes.length ? notes : null;
+}
+function withAttachmentNotes(text, notes) {
+    const trimmed = text.trim();
+    const usefulNotes = (notes ?? [])
+        .map((note) => note.text || note.note || (note.name ? `Attachment: ${note.name}` : ""))
+        .filter((note) => note.trim());
+    if (usefulNotes.length === 0)
+        return trimmed;
+    return [trimmed, ...usefulNotes].filter(Boolean).join("\n");
+}

package/dist/memory/lcm/schema.js

@@ -0,0 +1,188 @@
+const SCHEMA_VERSION = 7;
+export function runLcmMigrations(db) {
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    const run = () => {
+        db.exec(`
+			CREATE TABLE IF NOT EXISTS lcm_meta (
+				k TEXT PRIMARY KEY,
+				v TEXT NOT NULL
+			);
+
+			CREATE TABLE IF NOT EXISTS lcm_segments (
+				id TEXT PRIMARY KEY,
+				status TEXT NOT NULL DEFAULT 'active',
+				session_id TEXT,
+				channel_key TEXT,
+				started_at TEXT NOT NULL,
+				closed_at TEXT,
+				raw_pruned_at TEXT,
+				boundary_source_json TEXT,
+				metadata_json TEXT,
+				created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+				updated_at INTEGER NOT NULL DEFAULT (unixepoch())
+			);
+
+			CREATE TABLE IF NOT EXISTS lcm_records (
+				id INTEGER PRIMARY KEY,
+				record_key TEXT NOT NULL UNIQUE,
+				segment_id TEXT NOT NULL REFERENCES lcm_segments(id) ON DELETE CASCADE,
+				kind TEXT NOT NULL,
+				text_full TEXT NOT NULL,
+				parts_json TEXT,
+				happened_at TEXT NOT NULL,
+				session_id TEXT,
+				channel_key TEXT,
+				channel_id TEXT,
+				job_id TEXT,
+				source_type TEXT NOT NULL,
+				source_path TEXT,
+				source_line INTEGER,
+				source_record_id TEXT,
+				source_message_id TEXT,
+				source_ref TEXT,
+				attachments_json TEXT,
+				metadata_json TEXT,
+				created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+				updated_at INTEGER NOT NULL DEFAULT (unixepoch())
+			);
+
+			CREATE INDEX IF NOT EXISTS idx_lcm_records_segment ON lcm_records(segment_id, happened_at, id);
+			CREATE INDEX IF NOT EXISTS idx_lcm_records_source ON lcm_records(source_type, source_path, source_record_id);
+			CREATE INDEX IF NOT EXISTS idx_lcm_records_session ON lcm_records(session_id, channel_key, happened_at);
+
+			CREATE VIRTUAL TABLE IF NOT EXISTS lcm_records_fts USING fts5(
+				text_full,
+				content='',
+				contentless_delete=1
+			);
+
+			CREATE TABLE IF NOT EXISTS lcm_summaries (
+				id INTEGER PRIMARY KEY,
+				summary_key TEXT NOT NULL UNIQUE,
+				segment_id TEXT NOT NULL REFERENCES lcm_segments(id) ON DELETE CASCADE,
+				depth INTEGER NOT NULL,
+				status TEXT NOT NULL DEFAULT 'placeholder',
+				text_full TEXT NOT NULL,
+				pinned INTEGER NOT NULL DEFAULT 0,
+				covers_from_record_id INTEGER REFERENCES lcm_records(id) ON DELETE SET NULL,
+				covers_to_record_id INTEGER REFERENCES lcm_records(id) ON DELETE SET NULL,
+				snapshot_json TEXT,
+				source_type TEXT NOT NULL,
+				source_path TEXT,
+				source_line INTEGER,
+				source_record_id TEXT,
+				source_message_id TEXT,
+				source_ref TEXT,
+				metadata_json TEXT,
+				created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+				updated_at INTEGER NOT NULL DEFAULT (unixepoch())
+			);
+
+			CREATE INDEX IF NOT EXISTS idx_lcm_summaries_segment ON lcm_summaries(segment_id, depth, id);
+			CREATE INDEX IF NOT EXISTS idx_lcm_summaries_status ON lcm_summaries(status, pinned);
+
+			CREATE VIRTUAL TABLE IF NOT EXISTS lcm_summaries_fts USING fts5(
+				text_full,
+				content='',
+				contentless_delete=1
+			);
+
+			CREATE TABLE IF NOT EXISTS lcm_summary_sources (
+				summary_id INTEGER NOT NULL REFERENCES lcm_summaries(id) ON DELETE CASCADE,
+				ord INTEGER NOT NULL,
+				record_id INTEGER REFERENCES lcm_records(id) ON DELETE SET NULL,
+				-- Advisory legacy edge only. New condensation code must write summary-to-summary
+				-- lineage to lcm_summary_parents so parent coverage survives source-row drift.
+				source_summary_id INTEGER REFERENCES lcm_summaries(id) ON DELETE SET NULL,
+				source_ref TEXT,
+				snapshot_json TEXT,
+				PRIMARY KEY(summary_id, ord)
+			);
+
+			CREATE TABLE IF NOT EXISTS lcm_summary_parents (
+				summary_id INTEGER NOT NULL REFERENCES lcm_summaries(id) ON DELETE CASCADE,
+				parent_summary_id INTEGER NOT NULL REFERENCES lcm_summaries(id) ON DELETE CASCADE,
+				ord INTEGER NOT NULL,
+				PRIMARY KEY(summary_id, parent_summary_id)
+			);
+
+			CREATE INDEX IF NOT EXISTS idx_summary_parents_parent ON lcm_summary_parents(parent_summary_id);
+			CREATE INDEX IF NOT EXISTS idx_summary_parents_child ON lcm_summary_parents(summary_id);
+
+			CREATE TABLE IF NOT EXISTS lcm_context_items (
+				session_key TEXT NOT NULL,
+				ordinal INTEGER NOT NULL,
+				item_type TEXT NOT NULL CHECK(item_type IN ('raw', 'summary')),
+				record_id INTEGER REFERENCES lcm_records(id) ON DELETE CASCADE,
+				summary_id INTEGER REFERENCES lcm_summaries(id) ON DELETE CASCADE,
+				fingerprint TEXT NOT NULL,
+				happened_at TEXT,
+				updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
+				PRIMARY KEY(session_key, ordinal),
+				CHECK ((item_type = 'raw' AND record_id IS NOT NULL AND summary_id IS NULL)
+					OR (item_type = 'summary' AND summary_id IS NOT NULL AND record_id IS NULL))
+			);
+
+			CREATE INDEX IF NOT EXISTS idx_lcm_context_items_session ON lcm_context_items(session_key);
+			CREATE INDEX IF NOT EXISTS idx_lcm_context_items_record ON lcm_context_items(record_id);
+			CREATE INDEX IF NOT EXISTS idx_lcm_context_items_summary ON lcm_context_items(summary_id);
+
+			CREATE TABLE IF NOT EXISTS lcm_session_state (
+				session_key TEXT PRIMARY KEY,
+				compaction_debt INTEGER NOT NULL DEFAULT 0,
+				cache_touched_at INTEGER,
+				updated_at INTEGER
+			);
+		`);
+        migrateRecordPartsColumn(db);
+        migrateSummarySnapshotColumn(db);
+        migrateContentlessFts(db);
+        writeMeta(db, "schema_version", String(SCHEMA_VERSION));
+    };
+    if (db.inTransaction)
+        run();
+    else
+        db.transaction(run).immediate();
+}
+export function readMeta(db, key) {
+    const row = db.prepare("SELECT v FROM lcm_meta WHERE k = ?").get(key);
+    return row?.v ?? null;
+}
+export function writeMeta(db, key, value) {
+    db.prepare(`INSERT INTO lcm_meta(k, v) VALUES (?, ?)
+		 ON CONFLICT(k) DO UPDATE SET v = excluded.v`).run(key, value);
+}
+function migrateContentlessFts(db) {
+    migrateOneFts(db, "lcm_records_fts", "lcm_records", "text_full");
+    migrateOneFts(db, "lcm_summaries_fts", "lcm_summaries", "text_full");
+}
+function migrateRecordPartsColumn(db) {
+    const columns = db.prepare("PRAGMA table_info(lcm_records)").all();
+    if (columns.some((column) => column.name === "parts_json"))
+        return;
+    db.prepare("ALTER TABLE lcm_records ADD COLUMN parts_json TEXT").run();
+}
+function migrateSummarySnapshotColumn(db) {
+    const columns = db.prepare("PRAGMA table_info(lcm_summaries)").all();
+    if (columns.some((column) => column.name === "snapshot_json"))
+        return;
+    db.prepare("ALTER TABLE lcm_summaries ADD COLUMN snapshot_json TEXT").run();
+}
+function migrateOneFts(db, ftsTable, sourceTable, textColumn) {
+    const row = db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(ftsTable);
+    if (row?.sql.includes("contentless_delete=1"))
+        return;
+    const run = () => {
+        db.prepare(`DROP TABLE ${ftsTable}`).run();
+        db.prepare(`CREATE VIRTUAL TABLE ${ftsTable} USING fts5(${textColumn}, content='', contentless_delete=1)`).run();
+        const rows = db.prepare(`SELECT id, ${textColumn} FROM ${sourceTable}`).all();
+        const insert = db.prepare(`INSERT INTO ${ftsTable}(rowid, ${textColumn}) VALUES (?, ?)`);
+        for (const source of rows)
+            insert.run(source.id, source[textColumn]);
+    };
+    if (db.inTransaction)
+        run();
+    else
+        db.transaction(run).immediate();
+}

package/dist/memory/lcm/segment-manager.js

@@ -0,0 +1,136 @@
+import { projectNormalizedLcmBatch } from "./indexer.js";
+import { normalizeChatRecords } from "./normalize.js";
+export class LcmSegmentManager {
+    lcmStore;
+    memoryStore;
+    indexer;
+    newSessionRetainDepth;
+    onRotate;
+    activeSegments = new Map();
+    segmentCounters = new Map();
+    projectionQueue = Promise.resolve();
+    projectionFailures = 0;
+    constructor(options) {
+        this.lcmStore = options.lcmStore;
+        this.memoryStore = options.memoryStore;
+        this.indexer = options.indexer;
+        this.newSessionRetainDepth = options.newSessionRetainDepth;
+        this.onRotate = options.onRotate;
+    }
+    subscribeRuntime(runtime, sessionId) {
+        const unsubscribe = runtime.subscribe((record) => {
+            this.projectionQueue = this.projectionQueue.then(() => this.projectRuntimeRecord(runtime, record, sessionId), () => this.projectRuntimeRecord(runtime, record, sessionId));
+            void this.projectionQueue.catch((error) => {
+                this.projectionFailures += 1;
+                console.error(`memory projection failed for ${runtime.channelKey}`, error);
+            });
+        });
+        return unsubscribe;
+    }
+    async flush() {
+        await this.projectionQueue.catch(() => undefined);
+    }
+    stats() {
+        return { projectionFailures: this.projectionFailures };
+    }
+    activeSegmentId(channelKey) {
+        const existing = this.activeSegments.get(channelKey);
+        if (existing)
+            return existing;
+        const persisted = this.findPersistedActiveSegmentId(channelKey);
+        if (persisted) {
+            this.activeSegments.set(channelKey, persisted);
+            this.seedSegmentCounter(channelKey);
+            return persisted;
+        }
+        this.seedSegmentCounter(channelKey);
+        const next = this.nextSegmentId(channelKey);
+        this.activeSegments.set(channelKey, next);
+        return next;
+    }
+    async projectRuntimeRecord(runtime, record, sessionId) {
+        if (record.type === "runtime" && record.event === "reset") {
+            this.rotateRuntimeSegment(runtime, record);
+            return;
+        }
+        const segmentId = this.activeSegmentId(runtime.channelKey);
+        const batch = normalizeChatRecords([record], {
+            segmentId,
+            sessionId: sessionId ?? null,
+            channelKey: runtime.channelKey,
+        });
+        if (batch.records.length === 0 && batch.segments.length === 0)
+            return;
+        await projectNormalizedLcmBatch({ batch, lcmStore: this.lcmStore, indexer: this.indexer });
+    }
+    rotateRuntimeSegment(runtime, record) {
+        const previousSegmentId = this.activeSegmentId(runtime.channelKey);
+        const nextSegmentId = this.nextSegmentId(runtime.channelKey);
+        let indexDeletes = [];
+        // LCM rotation commits as one source-of-truth transaction. Shared-index deletes are
+        // applied after commit; startup reconciliation repairs crashes between the two DBs.
+        this.lcmStore.db
+            .transaction(() => {
+            const batch = normalizeChatRecords([record], {
+                segmentId: previousSegmentId,
+                channelKey: runtime.channelKey,
+            });
+            for (const segment of batch.segments)
+                this.lcmStore.ensureSegment(segment);
+            for (const normalizedRecord of batch.records)
+                this.lcmStore.insertRecord(normalizedRecord);
+            this.lcmStore.closeSegment(previousSegmentId, record.ts);
+            this.lcmStore.ensureSegment({
+                id: nextSegmentId,
+                channelKey: runtime.channelKey,
+                startedAt: record.ts,
+                boundarySource: {
+                    sourceType: "chat",
+                    sourceRecordId: record.recordId,
+                    sourceRef: `chat:${record.recordId}`,
+                },
+            });
+            this.lcmStore.clearContextItems(runtime.channelKey);
+            this.lcmStore.clearSessionState(runtime.channelKey);
+            const retention = this.lcmStore.applyNewSessionRetention({
+                newSessionRetainDepth: this.newSessionRetainDepth,
+                activeSegmentId: nextSegmentId,
+            });
+            indexDeletes = retention.indexDeletes;
+        })
+            .immediate();
+        this.activeSegments.set(runtime.channelKey, nextSegmentId);
+        this.onRotate?.(runtime.channelKey);
+        this.memoryStore.db
+            .transaction((deletes) => {
+            for (const deleted of deletes)
+                this.memoryStore.deleteBySourceUnsafe(deleted.corpus, deleted.sourceId);
+        })
+            .immediate(indexDeletes);
+    }
+    nextSegmentId(channelKey) {
+        const next = (this.segmentCounters.get(channelKey) ?? 0) + 1;
+        this.segmentCounters.set(channelKey, next);
+        return `${channelKey}:seg-${next}`;
+    }
+    findPersistedActiveSegmentId(channelKey) {
+        const segments = this.lcmStore
+            .listSegments()
+            .filter((segment) => segment.channelKey === channelKey && segment.status === "active")
+            .sort((a, b) => b.startedAt.localeCompare(a.startedAt) || b.id.localeCompare(a.id));
+        return segments[0]?.id ?? null;
+    }
+    seedSegmentCounter(channelKey) {
+        const existing = this.segmentCounters.get(channelKey) ?? 0;
+        let max = existing;
+        for (const segment of this.lcmStore.listSegments()) {
+            if (segment.channelKey !== channelKey)
+                continue;
+            const match = segment.id.match(/:seg-(\d+)$/);
+            if (!match)
+                continue;
+            max = Math.max(max, Number(match[1]));
+        }
+        this.segmentCounters.set(channelKey, max);
+    }
+}