@qearlyao/familiar 0.2.5 → 0.3.0

package/dist/ids.js

@@ -0,0 +1,11 @@
+import { randomUUID } from "node:crypto";
+export function toUnixMs(ts) {
+    const parsed = ts ? Date.parse(ts) : NaN;
+    return Number.isFinite(parsed) ? parsed : Date.now();
+}
+export function eventId() {
+    return `evt_${randomUUID()}`;
+}
+export function messageId(prefix = "msg") {
+    return `${prefix}_${randomUUID()}`;
+}

package/dist/index.js

@@ -1,4 +1,5 @@
 export { createFamiliarAgent } from "./agent.js";
+export { createAgentCore } from "./agent-core.js";
 export { buildRecordBase, chatChannelKey, chatLogPath, createChatLog, } from "./chat-log.js";
 export { loadConfig } from "./config.js";
 export { startDiscordDaemon } from "./discord.js";

package/dist/memory/index/store.js

@@ -11,6 +11,11 @@ export class MemoryIndexStore {
     embeddingProvider;
     embeddingModel;
     embeddingDimensions;
+    vectorCapabilityValue;
+    findChunkIdByHashStmt;
+    insertChunkStmt;
+    insertFtsStmt;
+    insertMemoryVecStmt;
     constructor(options) {
         if (!options.db && !options.path)
             throw new Error("MemoryIndexStore requires a db or path");
@@ -32,6 +37,16 @@ export class MemoryIndexStore {
             embeddingModel: this.embeddingModel,
             embeddingDimensions: this.embeddingDimensions,
         });
+        this.vectorCapabilityValue = readMeta(this.db, "vector_capability") === "sqlite-vec" ? "sqlite-vec" : "blob-js";
+        this.findChunkIdByHashStmt = this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?");
+        this.insertChunkStmt = this.db.prepare(`INSERT INTO memory_chunks (
+					content_hash, corpus, text_full, snippet, token_count, metadata_json, embedding_model,
+					embedding_dimensions, embedding
+				 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+        this.insertFtsStmt = this.db.prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)");
+        if (this.vectorCapabilityValue === "sqlite-vec") {
+            this.insertMemoryVecStmt = this.db.prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)");
+        }
     }
     static open(config) {
         return new MemoryIndexStore({
@@ -80,9 +95,7 @@ export class MemoryIndexStore {
                     this.insertSourceMapping(preloadedId, item);
                     continue;
                 }
-                const existing = this.db
-                    .prepare("SELECT id FROM memory_chunks WHERE content_hash = ?")
-                    .get(item.contentHash);
+                const existing = this.findChunkIdByHashStmt.get(item.contentHash);
                 if (existing)
                     this.insertSourceMapping(existing.id, item);
             }
@@ -333,7 +346,7 @@ export class MemoryIndexStore {
         };
     }
     vectorCapability() {
-        return readMeta(this.db, "vector_capability") === "sqlite-vec" ? "sqlite-vec" : "blob-js";
+        return this.vectorCapabilityValue;
     }
     vectorRowCount() {
         try {
@@ -377,26 +390,17 @@ export class MemoryIndexStore {
             ? { id: knownId }
             : knownMissingHashes?.has(item.contentHash)
                 ? undefined
-                : this.db.prepare("SELECT id FROM memory_chunks WHERE content_hash = ?").get(item.contentHash);
+                : this.findChunkIdByHashStmt.get(item.contentHash);
         if (existing) {
             knownIds.set(item.contentHash, existing.id);
             this.insertSourceMapping(existing.id, item);
             return existing.id;
         }
-        const result = this.db
-            .prepare(`INSERT INTO memory_chunks (
-					content_hash, corpus, text_full, snippet, token_count, metadata_json, embedding_model,
-					embedding_dimensions, embedding
-				 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
-            .run(item.contentHash, item.corpus, item.text, item.snippet, item.tokenCount, item.metadataJson, this.embeddingModel, this.embeddingDimensions, encodeVector(item.embedding));
+        const result = this.insertChunkStmt.run(item.contentHash, item.corpus, item.text, item.snippet, item.tokenCount, item.metadataJson, this.embeddingModel, this.embeddingDimensions, encodeVector(item.embedding));
         const id = Number(result.lastInsertRowid);
-        this.db
-            .prepare("INSERT INTO memory_fts(rowid, text_full, snippet) VALUES (?, ?, ?)")
-            .run(id, item.text, item.snippet);
+        this.insertFtsStmt.run(id, item.text, item.snippet);
         if (this.vectorCapability() === "sqlite-vec") {
-            this.db
-                .prepare("INSERT INTO memory_vec(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)")
-                .run(id, encodeVector(item.embedding));
+            this.insertMemoryVecStmt.run(id, encodeVector(item.embedding));
         }
         knownIds.set(item.contentHash, id);
         this.insertSourceMapping(id, item);

package/dist/memory/index/vector-codec.js

@@ -15,8 +15,8 @@ export function cosineDistance(a, b) {
     let aNorm = 0;
     let bNorm = 0;
     for (let index = 0; index < a.length; index++) {
-        const av = a[index] ?? 0;
-        const bv = b[index] ?? 0;
+        const av = a[index];
+        const bv = b[index];
         dot += av * bv;
         aNorm += av * av;
         bNorm += bv * bv;

package/dist/memory/lcm/context-transformer.js

@@ -176,8 +176,12 @@ export class LcmContextTransformer {
             tokensSaved = Math.max(0, candidate.chunkTokens - summaryItem.tokens);
             await this.condenseRuntimeSummaries({ state, sessionKey: input.sessionKey, signal: input.signal });
         };
-        input.state.compactionQueue = input.state.compactionQueue.then(run, run);
-        await input.state.compactionQueue;
+        // Serialize compactions per state: run after the prior one settles. Keep the shared
+        // compactionQueue non-rejecting so a failure here can't poison the next caller's link,
+        // and await the real work promise so this caller still surfaces its own error.
+        const settled = input.state.compactionQueue.then(run);
+        input.state.compactionQueue = settled.catch(() => undefined);
+        await settled;
         return { compacted, tokensSaved };
     }
     async persistRuntimeSummary(input) {

package/dist/memory/lcm/segment-manager.js

@@ -19,8 +19,12 @@ export class LcmSegmentManager {
     }
     subscribeRuntime(runtime, sessionId) {
         const unsubscribe = runtime.subscribe((record) => {
-            this.projectionQueue = this.projectionQueue.then(() => this.projectRuntimeRecord(runtime, record, sessionId), () => this.projectRuntimeRecord(runtime, record, sessionId));
-            void this.projectionQueue.catch((error) => {
+            // Serialize projections; the prior link is always non-rejecting (see the .catch
+            // below), so each projection runs once after it. Count + log failures in the same
+            // chain and recover, so the stored queue never carries a rejection forward.
+            this.projectionQueue = this.projectionQueue
+                .then(() => this.projectRuntimeRecord(runtime, record, sessionId))
+                .catch((error) => {
                 this.projectionFailures += 1;
                 console.error(`memory projection failed for ${runtime.channelKey}`, error);
             });

package/dist/memory/lcm/store/index-ids.js

@@ -0,0 +1,6 @@
+export function lcmRecordIndexSourceId(id) {
+    return `lcm_record:${id}`;
+}
+export function lcmSummaryIndexSourceId(id) {
+    return `lcm_summary:${id}`;
+}

package/dist/memory/lcm/store/inserts.js

@@ -0,0 +1,31 @@
+import { jsonOrNull, sourceRecordIdToString } from "./serialization.js";
+export function insertRecordPrepared(db, normalized) {
+    const inserted = db
+        .prepare(`INSERT INTO lcm_records (
+				record_key, segment_id, kind, text_full, happened_at, session_id, channel_key,
+				channel_id, job_id, source_type, source_path, source_line, source_record_id,
+				source_message_id, source_ref, attachments_json, metadata_json, parts_json
+			 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+        .run(normalized.recordKey, normalized.segmentId, normalized.kind, normalized.text, normalized.happenedAt, normalized.sessionId, normalized.channelKey, normalized.channelId, normalized.jobId, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.attachments), jsonOrNull(normalized.metadata), jsonOrNull(normalized.parts));
+    const id = Number(inserted.lastInsertRowid);
+    if (normalized.kind !== "boundary") {
+        db.prepare("INSERT INTO lcm_records_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
+    }
+    const row = db.prepare("SELECT * FROM lcm_records WHERE id = ?").get(id);
+    if (!row)
+        throw new Error(`Failed to read inserted LCM record: ${id}`);
+    return row;
+}
+export function insertSummaryPrepared(db, normalized, insertEdges) {
+    const inserted = db
+        .prepare(`INSERT INTO lcm_summaries (
+				summary_key, segment_id, depth, status, text_full, pinned,
+				covers_from_record_id, covers_to_record_id, snapshot_json, source_type, source_path,
+				source_line, source_record_id, source_message_id, source_ref, metadata_json
+			 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+        .run(normalized.summaryKey, normalized.segmentId, normalized.depth, normalized.status, normalized.text, normalized.pinned ? 1 : 0, normalized.coversFromRecordId, normalized.coversToRecordId, null, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.metadata));
+    const id = Number(inserted.lastInsertRowid);
+    db.prepare("INSERT INTO lcm_summaries_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
+    insertEdges(id, normalized.sourceItems, normalized.parents);
+    return id;
+}

package/dist/memory/lcm/store/normalizers.js

@@ -0,0 +1,91 @@
+import { stableHash } from "./serialization.js";
+export function normalizeSource(source) {
+    return {
+        sourceType: source.sourceType,
+        sourcePath: source.sourcePath ?? null,
+        sourceLine: source.sourceLine ?? null,
+        sourceRecordId: source.sourceRecordId ?? null,
+        sourceMessageId: source.sourceMessageId ?? null,
+        sourceRef: source.sourceRef ?? null,
+    };
+}
+export function normalizeRecordInput(input) {
+    const text = (input.text ?? "").trim();
+    if (!text && input.kind !== "boundary")
+        throw new Error("LCM record text must not be empty");
+    const source = normalizeSource(input.source);
+    const happenedAt = input.happenedAt ?? new Date().toISOString();
+    const parts = input.parts?.length ? input.parts : null;
+    const normalizedText = text || "Session boundary";
+    return {
+        segmentId: input.segmentId,
+        kind: input.kind,
+        text: normalizedText,
+        parts,
+        happenedAt,
+        sessionId: input.sessionId ?? null,
+        channelKey: input.channelKey ?? null,
+        channelId: input.channelId ?? null,
+        jobId: input.jobId ?? null,
+        source,
+        attachments: input.attachments?.length ? input.attachments : null,
+        metadata: input.metadata ?? null,
+        recordKey: computeLcmRecordKey({ ...input, happenedAt }),
+    };
+}
+export function normalizeSummaryInput(input) {
+    const text = (input.text ?? "").trim();
+    const source = normalizeSource(input.source);
+    const status = input.status ?? (text ? "ready" : "placeholder");
+    const normalizedText = text || "";
+    return {
+        segmentId: input.segmentId,
+        depth: input.depth,
+        status,
+        text: normalizedText,
+        pinned: input.pinned ?? false,
+        coversFromRecordId: input.coversFromRecordId ?? null,
+        coversToRecordId: input.coversToRecordId ?? null,
+        source,
+        sourceItems: input.sourceItems ?? [],
+        parents: input.parents ?? [],
+        metadata: input.metadata ?? null,
+        summaryKey: stableHash({
+            segmentId: input.segmentId,
+            depth: input.depth,
+            status,
+            text: normalizedText,
+            coversFromRecordId: input.coversFromRecordId ?? null,
+            coversToRecordId: input.coversToRecordId ?? null,
+            source,
+            parents: input.parents ?? [],
+        }),
+    };
+}
+export function computeLcmRecordKey(input) {
+    const text = (input.text ?? "").trim();
+    if (!text && input.kind !== "boundary")
+        throw new Error("LCM record text must not be empty");
+    const parts = input.parts?.length ? input.parts : null;
+    return stableHash({
+        segmentId: input.segmentId,
+        kind: input.kind,
+        text: text || "Session boundary",
+        parts,
+        happenedAt: input.happenedAt ?? new Date().toISOString(),
+        source: normalizeSource(input.source),
+    });
+}
+export function dedupeSummaryParentIds(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        if (!Number.isInteger(value) || value <= 0)
+            throw new Error("LCM summary parents must be positive integer ids");
+        if (seen.has(value))
+            continue;
+        seen.add(value);
+        result.push(value);
+    }
+    return result;
+}

package/dist/memory/lcm/store/row-mappers.js

@@ -0,0 +1,114 @@
+import { parseJsonArray, parseJsonObject } from "./serialization.js";
+function sourceFromRow(row) {
+    return {
+        sourceType: row.source_type,
+        sourcePath: row.source_path,
+        sourceLine: row.source_line,
+        sourceRecordId: row.source_record_id,
+        sourceMessageId: row.source_message_id,
+        sourceRef: row.source_ref,
+    };
+}
+export function segmentFromRow(row) {
+    return {
+        id: row.id,
+        status: row.status,
+        sessionId: row.session_id,
+        channelKey: row.channel_key,
+        startedAt: row.started_at,
+        closedAt: row.closed_at,
+        rawPrunedAt: row.raw_pruned_at,
+        boundarySource: parseJsonObject(row.boundary_source_json),
+        metadata: parseJsonObject(row.metadata_json),
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+export function recordFromRow(row) {
+    return {
+        id: row.id,
+        recordKey: row.record_key,
+        segmentId: row.segment_id,
+        kind: row.kind,
+        text: row.text_full,
+        parts: parseJsonArray(row.parts_json),
+        happenedAt: row.happened_at,
+        sessionId: row.session_id,
+        channelKey: row.channel_key,
+        channelId: row.channel_id,
+        jobId: row.job_id,
+        source: sourceFromRow(row),
+        attachments: parseJsonArray(row.attachments_json),
+        metadata: parseJsonObject(row.metadata_json),
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+export function summaryFromRow(row, parents = []) {
+    return {
+        id: row.id,
+        summaryKey: row.summary_key,
+        segmentId: row.segment_id,
+        depth: row.depth,
+        status: row.status,
+        text: row.text_full,
+        pinned: row.pinned === 1,
+        coversFromRecordId: row.covers_from_record_id,
+        coversToRecordId: row.covers_to_record_id,
+        source: sourceFromRow(row),
+        metadata: parseJsonObject(row.metadata_json),
+        snapshot: parseJsonArray(row.snapshot_json),
+        parents,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+export function summarySourceFromRow(row) {
+    return {
+        summaryId: row.summary_id,
+        ord: row.ord,
+        recordId: row.record_id,
+        sourceSummaryId: row.source_summary_id,
+        sourceRef: row.source_ref,
+        snapshot: parseJsonObject(row.snapshot_json),
+    };
+}
+export function contextItemFromRow(row) {
+    if (row.item_type === "raw") {
+        if (row.record_id === null)
+            throw new Error(`Invalid raw LCM context item at ordinal ${row.ordinal}`);
+        return {
+            sessionKey: row.session_key,
+            ordinal: row.ordinal,
+            type: "raw",
+            recordId: row.record_id,
+            summaryId: null,
+            fingerprint: row.fingerprint,
+            happenedAt: row.happened_at,
+            updatedAt: row.updated_at,
+        };
+    }
+    if (row.item_type === "summary") {
+        if (row.summary_id === null)
+            throw new Error(`Invalid summary LCM context item at ordinal ${row.ordinal}`);
+        return {
+            sessionKey: row.session_key,
+            ordinal: row.ordinal,
+            type: "summary",
+            recordId: null,
+            summaryId: row.summary_id,
+            fingerprint: row.fingerprint,
+            happenedAt: row.happened_at,
+            updatedAt: row.updated_at,
+        };
+    }
+    throw new Error(`Unknown LCM context item type: ${row.item_type}`);
+}
+export function sessionStateFromRow(row) {
+    return {
+        sessionKey: row.session_key,
+        compactionDebt: row.compaction_debt,
+        cacheTouchedAt: row.cache_touched_at,
+        updatedAt: row.updated_at,
+    };
+}

package/dist/memory/lcm/store/row-types.js

@@ -0,0 +1 @@
+export {};

package/dist/memory/lcm/store/serialization.js

@@ -0,0 +1,37 @@
+import { createHash } from "node:crypto";
+import { isRecord } from "../../../util/guards.js";
+export function jsonOrNull(value) {
+    if (value === null || value === undefined)
+        return null;
+    return JSON.stringify(value);
+}
+export function parseJsonObject(value) {
+    if (!value)
+        return null;
+    try {
+        const parsed = JSON.parse(value);
+        return isRecord(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function parseJsonArray(value) {
+    if (!value)
+        return null;
+    try {
+        const parsed = JSON.parse(value);
+        return Array.isArray(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function stableHash(value) {
+    return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}
+export function sourceRecordIdToString(value) {
+    if (value === null || value === undefined)
+        return null;
+    return String(value);
+}

package/dist/memory/lcm/store/snapshots.js

@@ -0,0 +1,73 @@
+import { parseJsonArray, parseJsonObject } from "./serialization.js";
+const SUMMARY_SNAPSHOT_TEXT_LIMIT = 4 * 1024;
+const SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX = "…[truncated]";
+export function buildSummarySnapshot(db, summary) {
+    const rows = db
+        .prepare(`SELECT * FROM lcm_records
+			 WHERE segment_id = ?
+			   AND id BETWEEN ? AND ?
+			 ORDER BY happened_at, id`)
+        .all(summary.segment_id, summary.covers_from_record_id, summary.covers_to_record_id);
+    return rows.map(snapshotRecordFromRow);
+}
+export function buildSummaryParentSnapshot(db, summaryId, visiting) {
+    if (visiting.has(summaryId))
+        throw new Error(`Cycle detected in LCM summary parents at ${summaryId}`);
+    visiting.add(summaryId);
+    const row = db.prepare("SELECT * FROM lcm_summaries WHERE id = ?").get(summaryId);
+    if (!row)
+        throw new Error(`LCM summary does not exist: ${summaryId}`);
+    let snapshot = parseJsonArray(row.snapshot_json);
+    if (!snapshot &&
+        row.covers_from_record_id !== null &&
+        row.covers_to_record_id !== null &&
+        db
+            .prepare("SELECT 1 FROM lcm_records WHERE segment_id = ? AND id BETWEEN ? AND ? LIMIT 1")
+            .get(row.segment_id, row.covers_from_record_id, row.covers_to_record_id)) {
+        snapshot = buildSummarySnapshot(db, row);
+    }
+    const parentRows = db
+        .prepare(`SELECT parent_summary_id
+			 FROM lcm_summary_parents
+			 WHERE summary_id = ?
+			 ORDER BY ord, parent_summary_id`)
+        .all(summaryId);
+    const parents = parentRows.map((parent) => buildSummaryParentSnapshot(db, parent.parent_summary_id, visiting));
+    visiting.delete(summaryId);
+    return {
+        summaryId: row.id,
+        depth: row.depth,
+        text: row.text_full,
+        coversFromRecordId: row.covers_from_record_id,
+        coversToRecordId: row.covers_to_record_id,
+        snapshot,
+        parents,
+    };
+}
+export function snapshotRecordFromRow(row) {
+    const metadata = parseJsonObject(row.metadata_json);
+    return {
+        id: row.id,
+        kind: row.kind,
+        happened_at: row.happened_at,
+        role: snapshotRole(row.kind, metadata),
+        text: truncateSummarySnapshotText(row.text_full),
+        parts: parseJsonArray(row.parts_json),
+        attachments: parseJsonArray(row.attachments_json),
+    };
+}
+export function snapshotRole(kind, metadata) {
+    if (typeof metadata?.role === "string" && metadata.role.trim())
+        return metadata.role;
+    if (kind === "user" || kind === "assistant")
+        return kind;
+    if (kind === "tool")
+        return "tool";
+    return null;
+}
+export function truncateSummarySnapshotText(text) {
+    if (text.length <= SUMMARY_SNAPSHOT_TEXT_LIMIT)
+        return text;
+    const retainedLength = Math.max(0, SUMMARY_SNAPSHOT_TEXT_LIMIT - SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX.length);
+    return `${text.slice(0, retainedLength)}${SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX}`;
+}