@qearlyao/familiar 0.1.0

package/dist/memory/operator.js

@@ -0,0 +1,477 @@
+import { existsSync, statSync } from "node:fs";
+import { mkdir, stat } from "node:fs/promises";
+import { resolve } from "node:path";
+import { stdin as input, stdout as output } from "node:process";
+import { createInterface } from "node:readline/promises";
+import { indexAllDiaryFiles } from "./diary/indexer.js";
+import { applyDoctorFixes, runDoctor } from "./doctor.js";
+import { ChunkIndexer } from "./index/chunk-indexer.js";
+import { createEmbeddingProvider } from "./index/embedding-provider.js";
+import { backfillFromChatLogs } from "./lcm/backfill.js";
+import { indexLcmRecords, indexLcmSummaries } from "./lcm/indexer.js";
+import { lcmRecordIndexSourceId, lcmSummaryIndexSourceId } from "./lcm/store.js";
+import { MemoryService } from "./service.js";
+export async function runMemoryOperator(config, argv) {
+    const [command, ...args] = argv;
+    if (!command || command === "--help" || command === "help") {
+        console.log(memoryHelp());
+        return;
+    }
+    switch (command) {
+        case "status":
+            await withOperatorService(config, async (service) => printStatus(config, service, args));
+            return;
+        case "doctor":
+            await withOperatorService(config, async (service) => runDoctorCommand(service, args));
+            return;
+        case "reindex":
+            await withOperatorService(config, async (service) => {
+                const { controller, dispose } = installMemoryAbortHandler("aborting reindex — finishing current corpus…");
+                try {
+                    await reindex(config, service, parseReindexArgs(args), undefined, controller.signal);
+                }
+                finally {
+                    dispose();
+                }
+            });
+            return;
+        case "backfill":
+            await withOperatorService(config, async (service) => backfill(config, service, parseBackfillArgs(config, args)));
+            return;
+        case "prune":
+            await withOperatorService(config, async (service) => prune(service, await parsePruneArgs(args)));
+            return;
+        case "backup":
+            await withOperatorService(config, async (service) => backup(config, service, parseBackupArgs(args)));
+            return;
+        default:
+            throw new Error(`Unknown memory subcommand: ${command}\n${memoryHelp()}`);
+    }
+}
+export function memoryHelp() {
+    return [
+        "Usage:",
+        "  familiar memory [workspace] status [--json]",
+        "  familiar memory [workspace] doctor [--clean]",
+        "  familiar memory [workspace] reindex [--corpus <name>] [--force]",
+        "  familiar memory [workspace] backfill [--channels <ch1,ch2>] [--data-dir <path>] [--dry-run]",
+        "  familiar memory [workspace] prune --new-session-retain-depth <N> [--yes] [--vacuum]",
+        "  familiar memory [workspace] backup <out-dir>",
+    ].join("\n");
+}
+async function withOperatorService(config, fn) {
+    const service = MemoryService.createWithoutRuntime(config);
+    try {
+        return await fn(service);
+    }
+    finally {
+        service.close();
+    }
+}
+function printStatus(config, service, args) {
+    const json = hasOnlyFlags(args, ["--json"]) && args.includes("--json");
+    const status = collectStatus(config, service);
+    if (json) {
+        console.log(JSON.stringify(status, null, 2));
+        return;
+    }
+    printPlainStatus(status);
+}
+function collectStatus(config, service) {
+    const lcmPath = resolve(config.memory.lcmDir, "lcm.sqlite");
+    const indexPath = resolve(config.memory.indexDir, "memory.sqlite");
+    const indexStats = service.memoryStore.stats();
+    return {
+        paths: {
+            lcm: { path: lcmPath, sizeBytes: fileSize(lcmPath) },
+            index: { path: indexPath, sizeBytes: fileSize(indexPath) },
+        },
+        counts: {
+            lcmRecords: countRows(service.lcmStore, "lcm_records"),
+            lcmSummariesByDepth: countGrouped(service.lcmStore, "SELECT depth AS key, COUNT(*) AS n FROM lcm_summaries GROUP BY depth"),
+            lcmSegments: {
+                active: countWhere(service.lcmStore, "lcm_segments", "status = 'active'"),
+                closed: countWhere(service.lcmStore, "lcm_segments", "status = 'closed'"),
+            },
+            lcmContextItems: countRows(service.lcmStore, "lcm_context_items"),
+            lcmSessionState: countRows(service.lcmStore, "lcm_session_state"),
+            memoryChunksByCorpus: countGrouped(service.memoryStore, "SELECT corpus AS key, COUNT(*) AS n FROM memory_chunks GROUP BY corpus"),
+            memoryIndexSources: countRows(service.memoryStore, "memory_index_sources"),
+            memoryFtsRows: indexStats.ftsRows,
+            memoryVectorRows: indexStats.vectorRows,
+        },
+        embedding: service.memoryStore.embeddingConfig(),
+        vector: {
+            capability: indexStats.vectorCapability,
+            available: indexStats.vectorAvailable,
+        },
+        projectionFailures: service.stats().projectionFailures,
+        requiresReindex: indexStats.requiresReindex,
+        schemaVersions: {
+            lcm: service.lcmStore.schemaVersion(),
+            index: readMemoryMeta(service.memoryStore, "schema_version"),
+        },
+    };
+}
+function runDoctorCommand(service, args) {
+    const clean = hasOnlyFlags(args, ["--clean"]) && args.includes("--clean");
+    const report = runDoctor({ lcm: service.lcmStore, index: service.memoryStore });
+    const projectionFailures = service.stats().projectionFailures;
+    if (projectionFailures > 0) {
+        report.findings.push({
+            kind: "projection_failures",
+            detail: `${projectionFailures} memory projection failure(s) swallowed in this process`,
+            fixable: false,
+        });
+        report.clean = false;
+    }
+    if (report.findings.length === 0) {
+        console.log("Memory doctor: clean");
+    }
+    else {
+        console.log(`Memory doctor: ${report.findings.length} finding(s)`);
+        for (const finding of report.findings)
+            console.log(`- ${formatFinding(finding)}`);
+    }
+    if (clean) {
+        const result = applyDoctorFixes({ lcm: service.lcmStore, index: service.memoryStore }, report);
+        console.log(result.summary);
+    }
+    if (!report.clean)
+        process.exitCode = 1;
+}
+async function reindex(config, service, options, embeddingProvider, signal) {
+    if (options.force) {
+        service.memoryStore.db
+            .prepare("DELETE FROM memory_meta WHERE k IN ('embedding_model', 'embedding_dimensions', 'requires_reindex')")
+            .run();
+    }
+    const corpora = options.corpus ? [options.corpus] : ["lcm_record", "lcm_summary", "diary_chunk"];
+    const runDelete = () => {
+        for (const corpus of corpora)
+            deleteCorpus(service.memoryStore, corpus);
+    };
+    if (service.memoryStore.db.inTransaction)
+        runDelete();
+    else
+        service.memoryStore.db.transaction(runDelete).immediate();
+    const indexer = options.force
+        ? new ChunkIndexer({
+            store: service.memoryStore,
+            embeddingProvider: embeddingProvider ?? createEmbeddingProvider(config),
+        })
+        : embeddingProvider
+            ? new ChunkIndexer({ store: service.memoryStore, embeddingProvider })
+            : service.indexer;
+    let chunks = 0;
+    if (corpora.includes("lcm_record")) {
+        if (signal?.aborted) {
+            console.log(`Reindexed ${chunks} chunk(s)`);
+            return;
+        }
+        const result = await indexLcmRecords({ indexer, records: service.lcmStore.listRecords(), signal });
+        chunks += result.ids.length;
+        printProgress(chunks);
+    }
+    if (corpora.includes("lcm_summary")) {
+        if (signal?.aborted) {
+            console.log(`Reindexed ${chunks} chunk(s)`);
+            return;
+        }
+        const result = await indexLcmSummaries({ indexer, summaries: service.lcmStore.listSummaries(), signal });
+        chunks += result.ids.length;
+        printProgress(chunks);
+    }
+    if (corpora.includes("diary_chunk")) {
+        if (signal?.aborted) {
+            console.log(`Reindexed ${chunks} chunk(s)`);
+            return;
+        }
+        const result = await indexAllDiaryFiles({ config, indexer, signal });
+        for (const file of result.files) {
+            if (signal?.aborted)
+                break;
+            chunks += file.result.ids.length;
+            printProgress(chunks);
+        }
+    }
+    console.log(`Reindexed ${chunks} chunk(s)`);
+}
+async function backfill(config, service, options) {
+    const embeddingProvider = createEmbeddingProvider(config);
+    const { controller, dispose } = installMemoryAbortHandler("aborting backfill — finishing current batch…");
+    try {
+        const report = await backfillFromChatLogs({
+            lcmStore: service.lcmStore,
+            memoryStore: service.memoryStore,
+            indexer: service.indexer,
+            embeddingProvider,
+            config,
+        }, { ...options, signal: controller.signal });
+        printBackfillReport(report);
+        if (report.errors.length > 0)
+            process.exitCode = 1;
+    }
+    finally {
+        dispose();
+    }
+}
+async function prune(service, options) {
+    if (!options.yes && !(await confirm(`Prune closed LCM raw records with retain depth ${options.retainDepth}?`))) {
+        console.log("Prune cancelled");
+        return;
+    }
+    const activeSegments = service.lcmStore.listSegments().filter((segment) => segment.status === "active");
+    const activeSegmentId = activeSegments.at(-1)?.id ?? null;
+    const runClose = () => {
+        for (const segment of activeSegments) {
+            if (segment.id !== activeSegmentId)
+                service.lcmStore.closeSegment(segment.id);
+        }
+    };
+    if (service.lcmStore.db.inTransaction)
+        runClose();
+    else
+        service.lcmStore.db.transaction(runClose).immediate();
+    const report = service.lcmStore.applyNewSessionRetention({
+        newSessionRetainDepth: options.retainDepth,
+        activeSegmentId,
+        vacuum: options.vacuum,
+    });
+    for (const ref of report.indexDeletes)
+        service.memoryStore.deleteBySource(ref.corpus, ref.sourceId);
+    const closedActive = activeSegments.filter((segment) => segment.id !== activeSegmentId);
+    if (closedActive.length > 0) {
+        const runReopen = () => {
+            for (const segment of closedActive) {
+                service.lcmStore.db
+                    .prepare("UPDATE lcm_segments SET status = 'active', closed_at = NULL, updated_at = unixepoch() WHERE id = ?")
+                    .run(segment.id);
+            }
+        };
+        if (service.lcmStore.db.inTransaction)
+            runReopen();
+        else
+            service.lcmStore.db.transaction(runReopen).immediate();
+    }
+    console.log(`Pruned ${report.rawRecordsDeleted} raw record(s), ${report.summariesDeleted} summary row(s), ` +
+        `${report.affectedSegments.length} closed segment(s) scanned`);
+}
+async function backup(config, service, outDir) {
+    await mkdir(outDir, { recursive: true });
+    const lcmOut = resolve(outDir, "lcm.sqlite");
+    const memoryOut = resolve(outDir, "memory.sqlite");
+    await Promise.all([service.lcmStore.db.backup(lcmOut), service.memoryStore.db.backup(memoryOut)]);
+    const [lcmStat, memoryStat] = await Promise.all([stat(lcmOut), stat(memoryOut)]);
+    console.log(`LCM backup: ${lcmOut} (${formatBytes(lcmStat.size)})`);
+    console.log(`Index backup: ${memoryOut} (${formatBytes(memoryStat.size)})`);
+    void config;
+}
+function parseReindexArgs(args) {
+    let corpus;
+    let force = false;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === "--force") {
+            force = true;
+            continue;
+        }
+        if (arg === "--corpus") {
+            corpus = args[++index];
+            if (!corpus)
+                throw new Error("Missing value for --corpus");
+            continue;
+        }
+        throw new Error(`Unknown reindex argument: ${arg}`);
+    }
+    return { corpus, force };
+}
+function parseBackfillArgs(config, args) {
+    let dataDir = config.workspace.dataDir;
+    let channels;
+    let dryRun = false;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === "--dry-run") {
+            dryRun = true;
+            continue;
+        }
+        if (arg === "--data-dir") {
+            const raw = args[++index];
+            if (!raw)
+                throw new Error("Missing value for --data-dir");
+            dataDir = resolve(raw);
+            continue;
+        }
+        if (arg === "--channels") {
+            const raw = args[++index];
+            if (!raw)
+                throw new Error("Missing value for --channels");
+            channels = raw
+                .split(",")
+                .map((channel) => channel.trim())
+                .filter(Boolean);
+            continue;
+        }
+        throw new Error(`Unknown backfill argument: ${arg}`);
+    }
+    return { dataDir, channels, dryRun };
+}
+async function parsePruneArgs(args) {
+    let retainDepth;
+    let yes = false;
+    let vacuum = false;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === "--yes") {
+            yes = true;
+            continue;
+        }
+        if (arg === "--vacuum") {
+            vacuum = true;
+            continue;
+        }
+        if (arg === "--new-session-retain-depth") {
+            const raw = args[++index];
+            if (!raw)
+                throw new Error("Missing value for --new-session-retain-depth");
+            retainDepth = Number(raw);
+            continue;
+        }
+        throw new Error(`Unknown prune argument: ${arg}`);
+    }
+    if (retainDepth === undefined || !Number.isInteger(retainDepth) || retainDepth < -1) {
+        throw new Error("prune requires --new-session-retain-depth <integer >= -1>");
+    }
+    return { retainDepth, yes, vacuum };
+}
+function parseBackupArgs(args) {
+    if (args.length !== 1)
+        throw new Error("backup requires <out-dir>");
+    return resolve(args[0]);
+}
+function hasOnlyFlags(args, allowed) {
+    for (const arg of args) {
+        if (!allowed.includes(arg))
+            throw new Error(`Unknown argument: ${arg}`);
+    }
+    return true;
+}
+function deleteCorpus(store, corpus) {
+    const rows = store.db.prepare("SELECT source_id FROM memory_index_sources WHERE corpus = ?").all(corpus);
+    const sourceIds = new Set(rows.map((row) => row.source_id));
+    for (const sourceId of sourceIds)
+        store.deleteBySourceUnsafe(corpus, sourceId);
+    const orphanRows = store.db.prepare("SELECT id FROM memory_chunks WHERE corpus = ?").all(corpus);
+    for (const row of orphanRows) {
+        store.db.prepare("DELETE FROM memory_fts WHERE rowid = ?").run(row.id);
+        store.db.prepare("DELETE FROM memory_chunks WHERE id = ?").run(row.id);
+    }
+}
+function installMemoryAbortHandler(message) {
+    const controller = new AbortController();
+    const onSigint = () => {
+        if (!controller.signal.aborted)
+            console.log(message);
+        controller.abort();
+    };
+    process.once("SIGINT", onSigint);
+    return {
+        controller,
+        dispose: () => process.off("SIGINT", onSigint),
+    };
+}
+function countRows(store, table) {
+    const row = store.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get();
+    return row.n;
+}
+function countWhere(store, table, where) {
+    const row = store.db.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE ${where}`).get();
+    return row.n;
+}
+function countGrouped(store, sql) {
+    const rows = store.db.prepare(sql).all();
+    return Object.fromEntries(rows.map((row) => [String(row.key), row.n]));
+}
+function readMemoryMeta(store, key) {
+    const row = store.db.prepare("SELECT v FROM memory_meta WHERE k = ?").get(key);
+    return row ? Number(row.v) : null;
+}
+function fileSize(path) {
+    return existsSync(path) ? statSync(path).size : 0;
+}
+function formatFinding(finding) {
+    return `${finding.kind} ${finding.fixable ? "[fixable]" : "[manual]"}: ${finding.detail}`;
+}
+function printPlainStatus(status) {
+    console.log("Memory status");
+    console.log(`LCM DB: ${status.paths.lcm.path} (${formatBytes(status.paths.lcm.sizeBytes)})`);
+    console.log(`Index DB: ${status.paths.index.path} (${formatBytes(status.paths.index.sizeBytes)})`);
+    console.log(`LCM records: ${status.counts.lcmRecords}`);
+    console.log(`LCM summaries by depth: ${JSON.stringify(status.counts.lcmSummariesByDepth)}`);
+    console.log(`LCM segments: active=${status.counts.lcmSegments.active} closed=${status.counts.lcmSegments.closed}`);
+    console.log(`LCM context items: ${status.counts.lcmContextItems}`);
+    console.log(`LCM session state: ${status.counts.lcmSessionState}`);
+    console.log(`Memory chunks by corpus: ${JSON.stringify(status.counts.memoryChunksByCorpus)}`);
+    console.log(`Memory index sources: ${status.counts.memoryIndexSources}`);
+    console.log(`Memory FTS rows: ${status.counts.memoryFtsRows}`);
+    console.log(`Memory vector rows: ${status.counts.memoryVectorRows} (${status.vector.available ? status.vector.capability : "blob-js fallback"})`);
+    console.log(`Embedding: ${status.embedding.provider}/${status.embedding.model} dim=${status.embedding.dimensions}`);
+    console.log(`Projection failures: ${status.projectionFailures}`);
+    if (status.requiresReindex)
+        console.log("Reindex required: run familiar memory reindex --force");
+    console.log(`Schema versions: lcm=${status.schemaVersions.lcm ?? "unknown"} index=${status.schemaVersions.index ?? "unknown"}`);
+}
+function printProgress(chunks) {
+    if (chunks > 0 && chunks % 100 === 0)
+        console.log(`Reindexed ${chunks} chunk(s)`);
+}
+function printBackfillReport(report) {
+    const rows = [
+        ["chatFilesProcessed", report.chatFilesProcessed],
+        ["transcriptFilesProcessed", report.transcriptFilesProcessed],
+        ["recordsInserted", report.recordsInserted],
+        ["recordsSkippedDuplicate", report.recordsSkippedDuplicate],
+        ["segmentsCreated", report.segmentsCreated],
+        ["summariesInserted", report.summariesInserted],
+        ["indexedChunks", report.indexedChunks],
+        ["errors", report.errors.length],
+    ];
+    const width = Math.max(...rows.map(([field]) => field.length));
+    console.log("Memory backfill summary");
+    for (const [field, value] of rows)
+        console.log(`${field.padEnd(width)}  ${value}`);
+    for (const error of report.errors)
+        console.log(`error  ${error}`);
+}
+async function confirm(question) {
+    const rl = createInterface({ input, output });
+    try {
+        const answer = await rl.question(`${question} Type yes to continue: `);
+        return answer.trim().toLowerCase() === "yes";
+    }
+    finally {
+        rl.close();
+    }
+}
+function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    const units = ["KB", "MB", "GB", "TB"];
+    let value = bytes / 1024;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    return `${value.toFixed(value >= 10 ? 1 : 2)} ${units[unit]}`;
+}
+export const __memoryOperatorTest = {
+    collectStatus,
+    reindex,
+    prune,
+    backup,
+    backfill,
+    lcmRecordIndexSourceId,
+    lcmSummaryIndexSourceId,
+};

package/dist/memory/service.js

@@ -0,0 +1,202 @@
+import { watch } from "node:fs";
+import { basename, resolve } from "node:path";
+import { __ambientDiaryInjectorTest, AmbientDiaryInjector } from "./diary/ambient-injector.js";
+import { DIARY_INDEX_FILE_RE, indexAllDiaryFiles, indexDiaryFile, removeDiaryFileIndex } from "./diary/indexer.js";
+import { ChunkIndexer } from "./index/chunk-indexer.js";
+import { createEmbeddingProvider } from "./index/embedding-provider.js";
+import { MemoryIndexStore } from "./index/store.js";
+import { LcmContextTransformer } from "./lcm/context-transformer.js";
+import { LcmSegmentManager } from "./lcm/segment-manager.js";
+import { LcmStore } from "./lcm/store.js";
+import { DefaultLcmSummarizer } from "./lcm/summarizer.js";
+import { createMemoryTools } from "./tools.js";
+export function createMemoryService(config, options = {}) {
+    return new DefaultMemoryService(config, options);
+}
+export const MemoryService = {
+    createWithoutRuntime(config, options = {}) {
+        return new DefaultMemoryService(config, options);
+    },
+};
+class DefaultMemoryService {
+    config;
+    lcmStore;
+    memoryStore;
+    embeddingProvider;
+    indexer;
+    segmentManager;
+    contextTransformer;
+    ambientInjector;
+    diaryWatchDebounceMs;
+    diaryWatcher;
+    diaryWatchTimers = new Map();
+    constructor(config, options = {}) {
+        this.config = config;
+        this.lcmStore = LcmStore.open(config);
+        this.memoryStore = MemoryIndexStore.open(config);
+        this.reconcileSharedIndex();
+        this.embeddingProvider = createEmbeddingProvider(config);
+        this.indexer = new ChunkIndexer({ store: this.memoryStore, embeddingProvider: this.embeddingProvider });
+        this.segmentManager = new LcmSegmentManager({
+            lcmStore: this.lcmStore,
+            memoryStore: this.memoryStore,
+            indexer: this.indexer,
+            newSessionRetainDepth: config.memory.lcm.newSessionRetainDepth,
+            onRotate: (sessionKey) => this.contextTransformer.invalidateSession(sessionKey),
+        });
+        this.contextTransformer = new LcmContextTransformer({
+            settings: config.memory.lcm,
+            lcmStore: this.lcmStore,
+            indexer: this.indexer,
+            summarizer: options.summarizer ?? new DefaultLcmSummarizer(config),
+            segmentManager: this.segmentManager,
+            now: options.now,
+        });
+        this.ambientInjector = new AmbientDiaryInjector({
+            store: this.memoryStore,
+            embeddingProvider: this.embeddingProvider,
+            enabled: config.memory.ambient.enabled,
+            topK: config.memory.ambient.topK,
+            minQueryLength: config.memory.ambient.minQueryLength,
+            throttleSeconds: config.memory.ambient.throttleSeconds,
+            weightSimilarity: config.memory.ambient.weightSimilarity,
+            weightValence: config.memory.ambient.weightValence,
+            weightRecency: config.memory.ambient.weightRecency,
+            weightIntensity: config.memory.ambient.weightIntensity,
+        });
+        this.diaryWatchDebounceMs = options.diaryWatchDebounceMs ?? 3000;
+    }
+    async indexDiaries() {
+        await indexAllDiaryFiles({ config: this.config, indexer: this.indexer, store: this.memoryStore });
+    }
+    watchDiaries() {
+        if (this.diaryWatcher)
+            return;
+        try {
+            this.diaryWatcher = watch(this.config.memory.diariesDir, { persistent: true }, (_eventType, filename) => {
+                if (!filename)
+                    return;
+                this.scheduleDiaryIndex(String(filename));
+            });
+            this.scheduleDiaryCatchUpIndex();
+        }
+        catch (error) {
+            if (isEnoent(error)) {
+                console.info(`diary watcher found no diary directory at ${this.config.memory.diariesDir}; disabled`);
+                return;
+            }
+            throw error;
+        }
+        this.diaryWatcher.on("error", (error) => {
+            console.error("diary watcher failed", error);
+            this.diaryWatcher?.close();
+            this.diaryWatcher = undefined;
+        });
+    }
+    memoryTools() {
+        return createMemoryTools({ store: this.memoryStore, embeddingProvider: this.embeddingProvider });
+    }
+    subscribeRuntime(runtime, sessionId) {
+        return this.segmentManager.subscribeRuntime(runtime, sessionId);
+    }
+    async transformContext(messages, signal, options = {}) {
+        const compacted = await this.contextTransformer.transformLcmContext(messages, signal, options);
+        if (options.skipAmbient)
+            return compacted;
+        return this.ambientInjector.inject(compacted, signal, options.sessionKey ?? options.sessionId ?? "default");
+    }
+    async serviceCompactionDebt(sessionKey) {
+        await this.contextTransformer.serviceCompactionDebt(sessionKey);
+    }
+    close() {
+        this.diaryWatcher?.close();
+        this.diaryWatcher = undefined;
+        for (const timer of this.diaryWatchTimers.values())
+            clearTimeout(timer);
+        this.diaryWatchTimers.clear();
+        this.memoryStore.close();
+        this.lcmStore.close();
+    }
+    async flush() {
+        await this.segmentManager.flush();
+    }
+    stats() {
+        return { projectionFailures: this.segmentManager.stats().projectionFailures };
+    }
+    reconcileSharedIndex() {
+        this.memoryStore.reconcileSources((source) => {
+            if (source.corpus === "lcm_record") {
+                const id = parseIndexSourceId(source.sourceId, "lcm_record");
+                return id !== null && this.lcmStore.getRecord(id) !== null;
+            }
+            if (source.corpus === "lcm_summary") {
+                const id = parseIndexSourceId(source.sourceId, "lcm_summary");
+                return id !== null && this.lcmStore.getSummary(id) !== null;
+            }
+            return true;
+        });
+    }
+    scheduleDiaryIndex(filename) {
+        const sourceId = basename(filename);
+        if (!DIARY_INDEX_FILE_RE.test(sourceId))
+            return;
+        const existing = this.diaryWatchTimers.get(sourceId);
+        if (existing)
+            clearTimeout(existing);
+        const timer = setTimeout(() => {
+            this.diaryWatchTimers.delete(sourceId);
+            void this.indexDiarySource(sourceId);
+        }, this.diaryWatchDebounceMs);
+        this.diaryWatchTimers.set(sourceId, timer);
+    }
+    scheduleDiaryCatchUpIndex() {
+        const timerKey = "__all__";
+        const existing = this.diaryWatchTimers.get(timerKey);
+        if (existing)
+            clearTimeout(existing);
+        const timer = setTimeout(() => {
+            this.diaryWatchTimers.delete(timerKey);
+            void this.indexDiaries().catch((error) => console.error("diary watcher catch-up indexing failed", error));
+        }, this.diaryWatchDebounceMs);
+        this.diaryWatchTimers.set(timerKey, timer);
+    }
+    async indexDiarySource(sourceId) {
+        const path = resolve(this.config.memory.diariesDir, sourceId);
+        try {
+            const result = await indexDiaryFile({
+                config: this.config,
+                indexer: this.indexer,
+                store: this.memoryStore,
+                path,
+            });
+            if ("skipped" in result)
+                return;
+            if (process.env.DEBUG === "memory-index") {
+                console.error(JSON.stringify({
+                    event: "diary_index_file",
+                    sourceId,
+                    chunks: result.result.ids.length,
+                    embedded: result.result.embedded,
+                    reused: result.result.reused,
+                }));
+            }
+        }
+        catch (error) {
+            if (isEnoent(error)) {
+                await removeDiaryFileIndex({ config: this.config, indexer: this.indexer, path });
+                return;
+            }
+            console.error(`diary index failed: ${path}`, error);
+        }
+    }
+}
+function parseIndexSourceId(value, prefix) {
+    if (!value?.startsWith(`${prefix}:`))
+        return null;
+    const id = Number(value.slice(prefix.length + 1));
+    return Number.isInteger(id) && id > 0 ? id : null;
+}
+function isEnoent(error) {
+    return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
+}
+export const __memoryServiceTest = __ambientDiaryInjectorTest;