@memory-river/core 0.2.0

package/dist/transcript/keyword-search.d.ts

@@ -0,0 +1,9 @@
+export interface KeywordSearchCandidate<T> {
+    value: T;
+    text: string;
+    timestamp?: number | string | Date | null;
+}
+export declare function buildKeywordSearchTerms(keyword: string): string[];
+export declare function countKeywordMatches(text: string, keyword: string): number;
+export declare function matchesKeywordSearch(text: string, keyword: string): boolean;
+export declare function rankKeywordMatches<T>(candidates: readonly KeywordSearchCandidate<T>[], keyword: string, limit?: number): T[];

package/dist/transcript/keyword-search.js

@@ -0,0 +1,67 @@
+const COMMON_SINGLE_CJK_TOKENS = new Set([
+    '的',
+    '了',
+    '是',
+    '在',
+    '有',
+    '他',
+    '她',
+    '我',
+    '你',
+    '不',
+    '一',
+]);
+export function buildKeywordSearchTerms(keyword) {
+    return [...new Set(keyword
+            .split(/\s+/)
+            .map(token => token.trim().toLowerCase())
+            .filter(token => token.length > 0)
+            .filter(token => !/^[\x00-\x7f]$/.test(token))
+            .filter(token => !COMMON_SINGLE_CJK_TOKENS.has(token)))];
+}
+export function countKeywordMatches(text, keyword) {
+    const normalized = text.toLowerCase();
+    return buildKeywordSearchTerms(keyword)
+        .reduce((count, token) => count + Number(normalized.includes(token)), 0);
+}
+export function matchesKeywordSearch(text, keyword) {
+    return countKeywordMatches(text, keyword) > 0;
+}
+export function rankKeywordMatches(candidates, keyword, limit = Number.POSITIVE_INFINITY) {
+    const terms = buildKeywordSearchTerms(keyword);
+    if (terms.length === 0 || limit <= 0)
+        return [];
+    return candidates
+        .map((candidate, index) => {
+        const normalized = candidate.text.toLowerCase();
+        const matchCount = terms.reduce((count, term) => count + Number(normalized.includes(term)), 0);
+        return {
+            ...candidate,
+            index,
+            matchCount,
+            timestampMs: normalizeTimestamp(candidate.timestamp),
+        };
+    })
+        .filter(candidate => candidate.matchCount > 0)
+        .sort((a, b) => b.matchCount - a.matchCount
+        || b.timestampMs - a.timestampMs
+        || a.index - b.index)
+        .slice(0, limit)
+        .map(candidate => candidate.value);
+}
+function normalizeTimestamp(timestamp) {
+    if (typeof timestamp === 'number')
+        return Number.isFinite(timestamp) ? timestamp : 0;
+    if (timestamp instanceof Date) {
+        const value = timestamp.getTime();
+        return Number.isFinite(value) ? value : 0;
+    }
+    if (typeof timestamp === 'string') {
+        const numeric = Number(timestamp);
+        if (Number.isFinite(numeric))
+            return numeric;
+        const parsed = Date.parse(timestamp);
+        return Number.isFinite(parsed) ? parsed : 0;
+    }
+    return 0;
+}

package/dist/transcript/rehydrate-keyword.d.ts

@@ -0,0 +1,6 @@
+import type { TranscriptEntry } from './rehydrate.js';
+export declare function rehydrateByKeyword(transcriptsDir: string, keyword: string, opts?: {
+    sessionKey?: string;
+    limit?: number;
+    offset?: number;
+}): Promise<TranscriptEntry[]>;

package/dist/transcript/rehydrate-keyword.js

@@ -0,0 +1,29 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { rankKeywordMatches } from './keyword-search.js';
+const MAX_KEYWORD_RESULTS = 200;
+export async function rehydrateByKeyword(transcriptsDir, keyword, opts) {
+    if (!fs.existsSync(transcriptsDir))
+        return [];
+    const files = fs.readdirSync(transcriptsDir)
+        .filter(file => file.endsWith('.jsonl') && !file.endsWith('.idx'))
+        .filter(file => !opts?.sessionKey || file === `${opts.sessionKey}.jsonl` || file.startsWith(`${opts.sessionKey}.`))
+        .sort((a, b) => fs.statSync(path.join(transcriptsDir, b)).mtimeMs - fs.statSync(path.join(transcriptsDir, a)).mtimeMs)
+        .slice(0, opts?.sessionKey ? undefined : 10);
+    const candidates = [];
+    for (const file of files) {
+        for (const line of fs.readFileSync(path.join(transcriptsDir, file), 'utf8').split('\n')) {
+            if (!line.trim())
+                continue;
+            try {
+                const entry = JSON.parse(line);
+                const text = `${entry.user ?? ''} ${entry.assistant ?? ''}`.toLowerCase();
+                candidates.push({ value: entry, text, timestamp: entry.timestamp });
+            }
+            catch { }
+        }
+    }
+    const offset = opts?.offset ?? 0;
+    const limit = Math.min(opts?.limit ?? 10, MAX_KEYWORD_RESULTS);
+    return rankKeywordMatches(candidates, keyword).slice(offset, offset + limit);
+}

package/dist/transcript/rehydrate.d.ts

@@ -0,0 +1,33 @@
+/**
+ * rehydrate.ts — Memory Rehydrate System
+ *
+ * 根據濃縮時記錄的 sourceEntryIds，從 transcript JSONL 撈回原始對話。
+ * 使用 .idx sidecar 做 O(1) byte offset seek，避免全檔 scan。
+ *
+ * Fallback：當 entryId 查不到（舊檔或 .idx 損壞），走 line-scan 並打日誌。
+ */
+export interface TranscriptEntry {
+    entryId: number;
+    sessionId: string;
+    user: string;
+    assistant: string;
+    timestamp: number;
+}
+/**
+ * 根據 entry IDs 撈原文，前後各擴 bleed 筆記錄。
+ * 使用 .idx sidecar 做 byte offset seek（O(1) 查詢）。
+ *
+ * @param jsonlPath  transcript JSONL 檔案路徑
+ * @param entryIds    要撈的 entry ID 陣列
+ * @param bleed      往前後各擴多少筆記錄（預設 2）
+ * @returns TranscriptEntry[] — 不論找到多少筆都回傳（可為空陣列）
+ */
+export declare function rehydrate(jsonlPath: string, entryIds: number[], bleed?: number): Promise<TranscriptEntry[]>;
+/**
+ * Fallback：LLM 完全沒給 ID 時，用時間窗救援（走 legacy line-scan）。
+ *
+ * @param jsonlPath      transcript JSONL 檔案路徑
+ * @param centerTimestamp  中心時間（ISO string 或 Unix ms）
+ * @param windowMinutes   時間窗分鐘數（預設 30）
+ */
+export declare function rehydrateByTime(jsonlPath: string, centerTimestamp: string, windowMinutes?: number): Promise<TranscriptEntry[]>;

package/dist/transcript/rehydrate.js

@@ -0,0 +1,285 @@
+/**
+ * rehydrate.ts — Memory Rehydrate System
+ *
+ * 根據濃縮時記錄的 sourceEntryIds，從 transcript JSONL 撈回原始對話。
+ * 使用 .idx sidecar 做 O(1) byte offset seek，避免全檔 scan。
+ *
+ * Fallback：當 entryId 查不到（舊檔或 .idx 損壞），走 line-scan 並打日誌。
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+function getIdxPath(jsonlPath) {
+    return jsonlPath + '.idx';
+}
+function loadIdx(jsonlPath) {
+    const idxPath = getIdxPath(jsonlPath);
+    try {
+        if (fs.existsSync(idxPath)) {
+            const raw = fs.readFileSync(idxPath, 'utf-8');
+            return JSON.parse(raw);
+        }
+    }
+    catch { }
+    return {};
+}
+function getTranscriptPaths(jsonlPath) {
+    const dir = path.dirname(jsonlPath);
+    const base = path.basename(jsonlPath, '.jsonl');
+    let rotated = [];
+    try {
+        rotated = fs.readdirSync(dir)
+            .filter((file) => file.startsWith(`${base}.`) && file.endsWith('.jsonl'))
+            .map((file) => ({
+            index: Number.parseInt(file.slice(base.length + 1, -'.jsonl'.length), 10),
+            filePath: path.join(dir, file),
+        }))
+            .filter((file) => Number.isFinite(file.index))
+            .sort((a, b) => b.index - a.index);
+    }
+    catch { }
+    return [jsonlPath, ...rotated.map((file) => file.filePath)];
+}
+// ---------------------------------------------------------------
+// 核心工具：從 byte offset 讀取一行 JSONL
+// ---------------------------------------------------------------
+function readLineAt(fd, offset) {
+    const MAX_LINE_LENGTH = 100_000; // 防止極端行的無限迴圈
+    const buf = Buffer.alloc(MAX_LINE_LENGTH);
+    const bytesRead = fs.readSync(fd, buf, 0, MAX_LINE_LENGTH, offset);
+    if (bytesRead === 0)
+        return null;
+    const content = buf.toString('utf-8', 0, bytesRead);
+    const eol = content.indexOf('\n');
+    if (eol === -1)
+        return content.trim() || null;
+    return content.substring(0, eol).trim() || null;
+}
+// ---------------------------------------------------------------
+// rehydrate：根據 entryIds 撈原文（O(1) seek via .idx）
+// ---------------------------------------------------------------
+/**
+ * 根據 entry IDs 撈原文，前後各擴 bleed 筆記錄。
+ * 使用 .idx sidecar 做 byte offset seek（O(1) 查詢）。
+ *
+ * @param jsonlPath  transcript JSONL 檔案路徑
+ * @param entryIds    要撈的 entry ID 陣列
+ * @param bleed      往前後各擴多少筆記錄（預設 2）
+ * @returns TranscriptEntry[] — 不論找到多少筆都回傳（可為空陣列）
+ */
+export async function rehydrate(jsonlPath, entryIds, bleed = 2) {
+    if (!entryIds || entryIds.length === 0)
+        return [];
+    const transcriptPaths = getTranscriptPaths(jsonlPath);
+    if (!transcriptPaths.some((transcriptPath) => fs.existsSync(transcriptPath)))
+        return [];
+    const found = new Map(); // entryId → entry
+    const foundPaths = new Map();
+    const indexes = new Map();
+    // ── O(1) seek via .idx ──────────────────────────────
+    for (const transcriptPath of transcriptPaths) {
+        const idx = loadIdx(transcriptPath);
+        indexes.set(transcriptPath, idx);
+        const missingIds = entryIds.filter((eid) => !found.has(eid));
+        if (missingIds.length === 0 || Object.keys(idx).length === 0)
+            continue;
+        let fd = null;
+        try {
+            fd = fs.openSync(transcriptPath, 'r');
+            for (const eid of missingIds) {
+                const offset = idx[eid];
+                if (offset === undefined) {
+                    continue;
+                }
+                const line = readLineAt(fd, offset);
+                if (!line)
+                    continue;
+                try {
+                    const raw = JSON.parse(line);
+                    found.set(eid, {
+                        entryId: typeof raw.entryId === 'number' ? raw.entryId : eid,
+                        sessionId: raw.sessionId || '',
+                        user: raw.user || '',
+                        assistant: raw.assistant || '',
+                        timestamp: typeof raw.timestamp === 'number' ? raw.timestamp : 0,
+                    });
+                    foundPaths.set(eid, transcriptPath);
+                }
+                catch {
+                    console.warn(`[rehydrate] JSON parse failed for entryId=${eid}`);
+                }
+            }
+        }
+        finally {
+            if (fd !== null)
+                fs.closeSync(fd);
+        }
+    }
+    // ── Legacy fallback：所有輪替代的 .idx 都 miss 後才 line scan ──────────────
+    const missingIds = entryIds.filter((eid) => !found.has(eid));
+    if (missingIds.length > 0) {
+        console.warn(`[rehydrate] legacy_transcript_fallback: path=${jsonlPath}, ids=${missingIds.join(',')}`);
+        for (const transcriptPath of transcriptPaths) {
+            const legacyEntries = await legacyLineScan(transcriptPath, missingIds, bleed);
+            for (const entry of legacyEntries) {
+                if (missingIds.includes(entry.entryId)) {
+                    found.set(entry.entryId, entry);
+                    foundPaths.set(entry.entryId, transcriptPath);
+                }
+            }
+        }
+    }
+    const entryMap = new Map(found);
+    for (const transcriptPath of transcriptPaths) {
+        const entries = Array.from(found.entries())
+            .filter(([entryId]) => foundPaths.get(entryId) === transcriptPath)
+            .map(([, entry]) => entry);
+        if (entries.length === 0)
+            continue;
+        for (const entry of await expandBleed(transcriptPath, entries, bleed, indexes.get(transcriptPath) ?? {})) {
+            entryMap.set(entry.entryId, entry);
+        }
+    }
+    return Array.from(entryMap.values()).sort((a, b) => a.entryId - b.entryId);
+}
+// ---------------------------------------------------------------
+// rehydrateByTime：時間窗救援（完全沒拿到 entryIds 時的 fallback）
+// ---------------------------------------------------------------
+/**
+ * Fallback：LLM 完全沒給 ID 時，用時間窗救援（走 legacy line-scan）。
+ *
+ * @param jsonlPath      transcript JSONL 檔案路徑
+ * @param centerTimestamp  中心時間（ISO string 或 Unix ms）
+ * @param windowMinutes   時間窗分鐘數（預設 30）
+ */
+export async function rehydrateByTime(jsonlPath, centerTimestamp, windowMinutes = 30) {
+    const transcriptPaths = getTranscriptPaths(jsonlPath);
+    if (!transcriptPaths.some((transcriptPath) => fs.existsSync(transcriptPath)))
+        return [];
+    const centerMs = isNaN(Number(centerTimestamp))
+        ? new Date(centerTimestamp).getTime()
+        : Number(centerTimestamp);
+    if (isNaN(centerMs)) {
+        console.warn(`[rehydrateByTime] invalid timestamp: ${centerTimestamp}`);
+        return [];
+    }
+    const halfWindowMs = windowMinutes * 60 * 1000;
+    const since = centerMs - halfWindowMs;
+    const until = centerMs + halfWindowMs;
+    const results = [];
+    for (const transcriptPath of transcriptPaths) {
+        try {
+            const raw = fs.readFileSync(transcriptPath, 'utf-8');
+            const lines = raw.split('\n').filter(l => l.trim());
+            for (const line of lines) {
+                try {
+                    const entry = JSON.parse(line);
+                    if (!entry.timestamp)
+                        continue;
+                    if (entry.timestamp >= since && entry.timestamp <= until) {
+                        results.push({
+                            entryId: entry.entryId ?? -1,
+                            sessionId: entry.sessionId || '',
+                            user: entry.user || '',
+                            assistant: entry.assistant || '',
+                            timestamp: entry.timestamp,
+                        });
+                    }
+                }
+                catch { }
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return results.sort((a, b) => a.timestamp - b.timestamp);
+}
+// ---------------------------------------------------------------
+// 內部工具
+// ---------------------------------------------------------------
+/**
+ * 對已排序的 entry 陣列，往前向後擴充 bleed 筆記錄。
+ */
+async function expandBleed(jsonlPath, entries, bleed, index) {
+    if (entries.length === 0 || bleed <= 0)
+        return entries;
+    const hitIds = new Set(entries.map(e => e.entryId));
+    const minId = Math.min(...hitIds) - bleed;
+    const maxId = Math.max(...hitIds) + bleed;
+    const entryMap = new Map();
+    entries.forEach(e => entryMap.set(e.entryId, e));
+    const fd = fs.openSync(jsonlPath, 'r');
+    try {
+        for (let id = minId; id <= maxId; id++) {
+            if (entryMap.has(id))
+                continue; // 已有的跳過
+            const offset = index[id];
+            if (offset == null)
+                continue; // 不存在的 ID 跳過
+            const buf = Buffer.alloc(8192);
+            const bytesRead = fs.readSync(fd, buf, 0, buf.length, offset);
+            const nlIdx = buf.indexOf(0x0a);
+            const lineEnd = nlIdx === -1 ? bytesRead : nlIdx;
+            const line = buf.slice(0, lineEnd).toString('utf8');
+            try {
+                const parsed = JSON.parse(line);
+                entryMap.set(parsed.entryId, parsed);
+            }
+            catch { /* 壞行跳過 */ }
+        }
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    return [...entryMap.values()].sort((a, b) => a.entryId - b.entryId);
+}
+/**
+ * Legacy line-scan：當 .idx 不存在或找不到 ID 時使用。
+ * 按 entryId 找目標 + 附近鄰居。
+ */
+async function legacyLineScan(jsonlPath, targetIds, bleed) {
+    const targetSet = new Set(targetIds);
+    const found = [];
+    const allEntries = [];
+    try {
+        const raw = fs.readFileSync(jsonlPath, 'utf-8');
+        const lines = raw.split('\n').filter(l => l.trim());
+        for (const line of lines) {
+            try {
+                const entry = JSON.parse(line);
+                allEntries.push({
+                    entryId: entry.entryId ?? -1,
+                    sessionId: entry.sessionId || '',
+                    user: entry.user || '',
+                    assistant: entry.assistant || '',
+                    timestamp: entry.timestamp || 0,
+                });
+            }
+            catch { }
+        }
+    }
+    catch (err) {
+        console.error(`[rehydrate] legacyLineScan failed:`, err);
+        return [];
+    }
+    if (allEntries.length === 0)
+        return [];
+    // 對有 targetId 的精確找，沒有 targetId 就取所有（因為 legacy 檔根本沒 entryId）
+    if (targetIds.length > 0 && targetIds[0] !== -1) {
+        for (const entry of allEntries) {
+            if (targetSet.has(entry.entryId)) {
+                found.push(entry);
+            }
+        }
+        // 沒有精確匹配：說明是舊檔（沒 entryId），全部返回
+        if (found.length === 0) {
+            console.warn(`[rehydrate] legacy_transcript_fallback: no entryId match, returning all entries (${allEntries.length})`);
+            return allEntries.slice(-20); // 最多取最近 20 筆
+        }
+        return found;
+    }
+    else {
+        // 沒有 targetId → 全量返回（最多 20 筆）
+        return allEntries.slice(-20);
+    }
+}

package/dist/transcript/transcript-archive.d.ts

@@ -0,0 +1,46 @@
+/**
+ * transcript-archive.ts
+ *
+ * Raw transcript 持久化層 — memory-river 濃縮失敗的底層保障。
+ * Append-only JSONL，5MB 上限自動 rotation（留最多 10 個輪替檔）。
+ *
+ * 與 streaming-recovery/scripts/transcriptStore.ts 共用相同的 rotation 模式，
+ * 但專注於「即將被蒸餾的原始訊息」的快照，不依賴任何外部 library。
+ *
+ * [Rehydrate System]
+ * - 每筆 entry 自帶 entryId（單調遞增），寫入 .idx sidecar 做 O(1) seek
+ * - transcript.counter 持久化 counter 狀態
+ */
+import { ContextMessage } from '../types.js';
+/**
+ * 由 sessionIdentity 衍生的 archive 識別子。
+ *
+ * - canonicalKey：對外統一 key，用於 in-memory cache（Phase 4-2 起）
+ * - sessionKey：用於磁碟檔名（Q6 漸進路徑，新檔仍走 sessionKey 命名）
+ * - sessionId：4-4 disk read fallback 用（sessionKey-named 找不到時試 sessionId-named）
+ *
+ * 結構上等同 SessionIdentity 子集；caller 可以直接把 SessionIdentity 傳進來。
+ */
+export interface TranscriptIdentity {
+    canonicalKey: string;
+    sessionKey: string | null;
+    sessionId?: string | null;
+}
+export interface RawTranscriptEntry {
+    entryId: number;
+    user: string;
+    assistant: string;
+    timestamp: number;
+}
+export interface ArchiveSnapshotResult {
+    ok: boolean;
+    appendedEntries: number;
+    dedupSkipped: number;
+}
+export declare function createTranscriptArchive(transcriptsDir: string): {
+    archiveSnapshot: (identity: TranscriptIdentity, messages: ContextMessage[]) => ArchiveSnapshotResult;
+    clearTranscriptCache: () => void;
+    getRawTranscript: (identity: TranscriptIdentity, since?: number, limit?: number) => RawTranscriptEntry[];
+    getTranscriptPath: (sessionKey: string) => string;
+};
+export type TranscriptArchive = ReturnType<typeof createTranscriptArchive>;