@qearlyao/familiar 0.1.0

package/dist/memory/diary/ambient.js

@@ -0,0 +1,124 @@
+import { retrieveMemory, } from "../index/retrieval.js";
+import { DIARY_CHUNK_CORPUS } from "./chunks.js";
+const DEFAULT_LIMIT = 4;
+const DEFAULT_CANDIDATE_MULTIPLIER = 5;
+const DEFAULT_HALF_LIFE_DAYS = 45;
+const MAX_AMBIENT_SEMANTIC_DISTANCE = 0.38;
+const DEFAULT_WEIGHTS = {
+    similarity: 1.0,
+    valence: 0.08,
+    intensity: 0.1,
+    recency: 0.08,
+};
+export async function retrieveAmbientDiary(options) {
+    const limit = positiveIntegerOrDefault(options.limit, DEFAULT_LIMIT);
+    const candidateLimit = positiveIntegerOrDefault(options.candidateLimit, Math.max(limit * DEFAULT_CANDIDATE_MULTIPLIER, limit));
+    const hits = await retrieveMemory({
+        query: options.query,
+        store: options.store,
+        embeddingProvider: options.embeddingProvider,
+        scope: { corpora: [DIARY_CHUNK_CORPUS] },
+        limit: candidateLimit,
+        candidateLimit,
+        useLexical: options.useLexical,
+        useSemantic: options.useSemantic,
+        signal: options.signal,
+    });
+    const weights = { ...DEFAULT_WEIGHTS, ...options.metadataBoosts, ...options.weights };
+    const now = options.now ?? new Date();
+    const halfLifeDays = positiveIntegerOrDefault(options.recencyHalfLifeDays, DEFAULT_HALF_LIFE_DAYS);
+    const scored = hits.map((hit) => scoreAmbientHit(hit, { now, halfLifeDays, weights }));
+    const relevant = scored.filter(hasAmbientRelevance);
+    debugAmbientHits(options.query, scored, relevant);
+    return relevant.sort(compareAmbientHits).slice(0, limit);
+}
+function hasAmbientRelevance(hit) {
+    if (hit.semanticScore === null)
+        return hit.lexicalRank !== null;
+    return hit.semanticScore <= MAX_AMBIENT_SEMANTIC_DISTANCE;
+}
+function debugAmbientHits(query, scored, relevant) {
+    if (!process.env.DEBUG?.split(",")
+        .map((part) => part.trim())
+        .includes("memory-ambient"))
+        return;
+    const relevantIds = new Set(relevant.map((hit) => hit.id));
+    console.error(JSON.stringify({
+        event: "ambient_diary_hits",
+        query,
+        maxSemanticDistance: MAX_AMBIENT_SEMANTIC_DISTANCE,
+        hits: scored.map((hit) => ({
+            id: hit.id,
+            passed: relevantIds.has(hit.id),
+            ambientScore: hit.ambientScore,
+            lexicalRank: hit.lexicalRank,
+            semanticRank: hit.semanticRank,
+            lexicalScore: hit.lexicalScore,
+            semanticScore: hit.semanticScore,
+            snippet: (hit.chunk.snippet || hit.chunk.text).slice(0, 120),
+        })),
+    }));
+}
+function scoreAmbientHit(hit, options) {
+    const similarity = hit.score * options.weights.similarity;
+    const valence = normalizeValence(metadataNumber(hit.chunk, "valence")) * options.weights.valence;
+    const intensity = normalizeUnit(metadataNumber(hit.chunk, "intensity")) * options.weights.intensity;
+    const recency = recencyScore(hit.chunk, options.now, options.halfLifeDays) * options.weights.recency;
+    const ambientScore = similarity + valence + intensity + recency;
+    return {
+        ...hit,
+        ambientScore,
+        boosts: { similarity, valence, intensity, recency },
+    };
+}
+function compareAmbientHits(a, b) {
+    return (b.ambientScore - a.ambientScore ||
+        b.score - a.score ||
+        (a.semanticRank ?? Number.POSITIVE_INFINITY) - (b.semanticRank ?? Number.POSITIVE_INFINITY) ||
+        (a.lexicalRank ?? Number.POSITIVE_INFINITY) - (b.lexicalRank ?? Number.POSITIVE_INFINITY) ||
+        a.id - b.id);
+}
+function recencyScore(chunk, now, halfLifeDays) {
+    const date = metadataDate(chunk) ?? timestampDate(chunk.createdAt);
+    if (!date)
+        return 0;
+    const ageMs = Math.max(0, now.getTime() - date.getTime());
+    const ageDays = ageMs / 86_400_000;
+    return 0.5 ** (ageDays / halfLifeDays);
+}
+function metadataDate(chunk) {
+    const value = chunk.metadata?.date;
+    if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value))
+        return null;
+    const date = new Date(`${value}T00:00:00.000Z`);
+    return Number.isNaN(date.getTime()) ? null : date;
+}
+function timestampDate(value) {
+    if (!Number.isFinite(value) || value <= 0)
+        return null;
+    const milliseconds = value < 10_000_000_000 ? value * 1000 : value;
+    const date = new Date(milliseconds);
+    return Number.isNaN(date.getTime()) ? null : date;
+}
+function metadataNumber(chunk, key) {
+    const value = chunk.metadata?.[key];
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (typeof value === "string" && value.trim() && Number.isFinite(Number(value)))
+        return Number(value);
+    return null;
+}
+function normalizeValence(value) {
+    if (value === null)
+        return 0;
+    return Math.max(0, Math.min(1, value));
+}
+function normalizeUnit(value) {
+    if (value === null)
+        return 0;
+    const absolute = Math.abs(value);
+    return Math.max(0, Math.min(1, absolute > 1 ? absolute / 10 : absolute));
+}
+function positiveIntegerOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
+}

package/dist/memory/diary/chunks.js

@@ -0,0 +1,231 @@
+import { basename } from "node:path";
+export const DIARY_CHUNK_CORPUS = "diary_chunk";
+const DEFAULT_MAX_CHARS = 2400;
+const DIARY_DATE_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
+const FRONTMATTER_RE = /^\s*---\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|$)/;
+const METADATA_LINE_RE = /^\s*(?:<!--\s*)?@?([A-Za-z][A-Za-z0-9_-]*)\s*:\s*(.*?)\s*(?:-->)?\s*$/;
+const SUPPORTED_METADATA_KEYS = new Set(["date", "heading", "valence", "intensity"]);
+export function chunkDiaryMarkdown(markdown, options = {}) {
+    const sourceId = options.sourceId ?? "diary.md";
+    const sourceRef = options.sourceRef ?? sourceId;
+    const { body, metadata: documentMetadata } = stripFrontmatter(markdown);
+    const baseMetadata = normalizeMetadata({
+        date: options.date ?? dateFromSourceId(sourceId),
+        ...documentMetadata,
+        ...options.metadata,
+    });
+    const sections = splitMarkdownSections(body);
+    const chunks = [];
+    const maxChars = positiveIntegerOrDefault(options.maxChars, DEFAULT_MAX_CHARS);
+    for (const section of sections) {
+        const { metadata: sectionMetadata, lines } = peelMetadataLines(section.lines);
+        const text = lines.join("\n").trim();
+        if (!text)
+            continue;
+        const metadata = normalizeMetadata({
+            ...baseMetadata,
+            ...(section.heading ? { heading: section.heading } : {}),
+            ...sectionMetadata,
+        });
+        for (const part of splitDiaryTextBlocks(text, maxChars)) {
+            chunks.push({
+                text: part,
+                chunkIndex: chunks.length,
+                sourceId,
+                sourceRef,
+                metadata,
+                snippet: buildSnippet(part, metadata),
+            });
+        }
+    }
+    return chunks;
+}
+export function diaryChunksToIndexInputs(chunks) {
+    return chunks.map((chunk) => ({
+        corpus: DIARY_CHUNK_CORPUS,
+        sourceId: chunk.sourceId,
+        sourceRef: chunk.sourceRef,
+        chunkIndex: chunk.chunkIndex,
+        text: chunk.text,
+        snippet: chunk.snippet,
+        metadata: chunk.metadata,
+    }));
+}
+export async function indexDiaryMarkdown(options) {
+    const sourceId = options.sourceId ?? basename(options.path);
+    const sourceRef = options.sourceRef ?? options.path;
+    const chunks = chunkDiaryMarkdown(options.markdown, { ...options, sourceId, sourceRef });
+    return options.indexer.replaceSource(DIARY_CHUNK_CORPUS, sourceId, diaryChunksToIndexInputs(chunks), options.signal);
+}
+function stripFrontmatter(markdown) {
+    const match = FRONTMATTER_RE.exec(markdown);
+    if (!match)
+        return { body: markdown, metadata: {} };
+    return {
+        body: markdown.slice(match[0].length),
+        metadata: parseMetadataBlock(match[1] ?? ""),
+    };
+}
+function splitMarkdownSections(markdown) {
+    const lines = markdown.replace(/\r\n/g, "\n").split("\n");
+    const sections = [{ lines: [], startLine: 1 }];
+    for (const [lineIndex, line] of lines.entries()) {
+        const heading = /^(#{1,6})\s+(.+?)\s*$/.exec(line);
+        if (heading) {
+            const current = sections[sections.length - 1];
+            if (current?.lines.every((value) => !value.trim()))
+                current.lines = [];
+            sections.push({
+                heading: stripInlineMarkdown(heading[2] ?? ""),
+                level: (heading[1] ?? "").length,
+                lines: [line],
+                startLine: lineIndex + 1,
+            });
+            continue;
+        }
+        sections[sections.length - 1].lines.push(line);
+    }
+    return sections;
+}
+function peelMetadataLines(lines) {
+    const metadata = {};
+    const body = [];
+    let inMetadataLead = true;
+    for (const line of lines) {
+        if (isMarkdownHeading(line)) {
+            continue;
+        }
+        const parsed = parseMetadataLine(line);
+        if (inMetadataLead && parsed) {
+            metadata[parsed.key] = parsed.value;
+            continue;
+        }
+        if (line.trim())
+            inMetadataLead = false;
+        body.push(line);
+    }
+    return { metadata, lines: body };
+}
+function parseMetadataBlock(block) {
+    const metadata = {};
+    for (const line of block.split(/\r?\n/)) {
+        const parsed = parseMetadataLine(line);
+        if (parsed)
+            metadata[parsed.key] = parsed.value;
+    }
+    return metadata;
+}
+function parseMetadataLine(line) {
+    const match = METADATA_LINE_RE.exec(line);
+    if (!match)
+        return null;
+    const rawKey = (match[1] ?? "").trim();
+    const key = camelMetadataKey(rawKey);
+    if (!SUPPORTED_METADATA_KEYS.has(key))
+        return null;
+    const rawValue = (match[2] ?? "").trim();
+    return { key, value: parseMetadataValue(rawValue) };
+}
+function parseMetadataValue(value) {
+    const unquoted = value.replace(/^["']|["']$/g, "");
+    if (/^-?\d+(?:\.\d+)?$/.test(unquoted))
+        return Number(unquoted);
+    return unquoted;
+}
+function normalizeMetadata(metadata) {
+    const out = {};
+    for (const [key, value] of Object.entries(metadata)) {
+        if (value === undefined || value === null || value === "")
+            continue;
+        out[camelMetadataKey(key)] = value;
+    }
+    return out;
+}
+function camelMetadataKey(key) {
+    const lower = key.toLowerCase();
+    return lower.replace(/[-_]([a-z])/g, (_match, letter) => letter.toUpperCase());
+}
+function splitDiaryTextBlocks(text, maxChars) {
+    const chunks = [];
+    for (const block of collectDiaryBlocks(text)) {
+        chunks.push(...splitOversizedParagraph(block, maxChars));
+    }
+    return chunks;
+}
+function collectDiaryBlocks(text) {
+    const blocks = [];
+    let current = [];
+    let currentListIndent = null;
+    const flush = () => {
+        const block = current.join(" ").replace(/\s+/g, " ").trim();
+        if (block)
+            blocks.push(block);
+        current = [];
+        currentListIndent = null;
+    };
+    for (const rawLine of text.replace(/\r\n/g, "\n").split("\n")) {
+        const line = rawLine.replace(/\s+$/g, "");
+        if (!line.trim()) {
+            flush();
+            continue;
+        }
+        const listItem = /^(\s*)(?:[-*+]|\d+[.)])\s+(.+)$/.exec(line);
+        if (listItem) {
+            flush();
+            current = [(listItem[2] ?? "").trim()];
+            currentListIndent = (listItem[1] ?? "").length;
+            continue;
+        }
+        if (currentListIndent !== null && /^\s+/.test(line)) {
+            current.push(line.trim());
+            continue;
+        }
+        flush();
+        current = [line.trim()];
+    }
+    flush();
+    return blocks;
+}
+function splitOversizedParagraph(text, maxChars) {
+    if (text.length <= maxChars)
+        return [text];
+    const chunks = [];
+    for (let index = 0; index < text.length;) {
+        let end = Math.min(index + maxChars, text.length);
+        if (end < text.length && isHighSurrogate(text.charCodeAt(end - 1)))
+            end -= 1;
+        if (end <= index)
+            end = Math.min(index + maxChars, text.length);
+        const chunk = text.slice(index, end).trim();
+        if (chunk)
+            chunks.push(chunk);
+        index = end;
+    }
+    return chunks;
+}
+function isHighSurrogate(value) {
+    return value >= 0xd800 && value <= 0xdbff;
+}
+function buildSnippet(text, metadata) {
+    const prefix = [metadata.date, metadata.heading]
+        .filter((value) => typeof value === "string" && value.trim())
+        .join(" ");
+    const body = text.replace(/\s+/g, " ").trim().slice(0, 220);
+    return prefix ? `${prefix}: ${body}` : body;
+}
+function dateFromSourceId(sourceId) {
+    const match = DIARY_DATE_RE.exec(basename(sourceId));
+    return match?.[1];
+}
+function stripInlineMarkdown(value) {
+    return value
+        .replace(/^\s*#+\s*/, "")
+        .replace(/[*_`~[\]()]/g, "")
+        .trim();
+}
+function isMarkdownHeading(line) {
+    return /^#{1,6}\s+/.test(line);
+}
+function positiveIntegerOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
+}

package/dist/memory/diary/index.js

@@ -0,0 +1,3 @@
+export { retrieveAmbientDiary, } from "./ambient.js";
+export { chunkDiaryMarkdown, DIARY_CHUNK_CORPUS, diaryChunksToIndexInputs, indexDiaryMarkdown, } from "./chunks.js";
+export { DIARY_INDEX_FILE_RE, indexAllDiaryFiles, indexDiaryFile, isDatedDiaryMarkdownFile, listDiaryMarkdownFiles, removeDiaryFileIndex, } from "./indexer.js";

package/dist/memory/diary/indexer.js

@@ -0,0 +1,93 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, isAbsolute, join, resolve } from "node:path";
+import { DIARY_CHUNK_CORPUS, indexDiaryMarkdown } from "./chunks.js";
+export const DIARY_INDEX_FILE_RE = /^\d{4}-\d{2}-\d{2}\.md$/;
+export function isDatedDiaryMarkdownFile(path) {
+    return DIARY_INDEX_FILE_RE.test(basename(path));
+}
+export async function listDiaryMarkdownFiles(config) {
+    let entries;
+    try {
+        entries = await readdir(config.memory.diariesDir, { withFileTypes: true });
+    }
+    catch (error) {
+        if (isEnoent(error)) {
+            console.info(`diary indexer found no diary directory at ${config.memory.diariesDir}; indexed 0 files`);
+            return [];
+        }
+        throw error;
+    }
+    return entries
+        .filter((entry) => entry.isFile() && DIARY_INDEX_FILE_RE.test(entry.name))
+        .map((entry) => join(config.memory.diariesDir, entry.name))
+        .sort();
+}
+function isEnoent(error) {
+    return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
+}
+export async function indexDiaryFile(options) {
+    const path = resolveDiaryPath(options.config, options.path);
+    const sourceId = basename(path);
+    const skipInvalid = options.skipInvalid ?? true;
+    if (!isDatedDiaryMarkdownFile(path)) {
+        if (skipInvalid)
+            return { path, sourceId, skipped: true, reason: "not-dated-markdown" };
+        throw new Error(`Diary file must be named YYYY-MM-DD.md: ${path}`);
+    }
+    const fileStat = await stat(path);
+    if (!fileStat.isFile()) {
+        if (skipInvalid)
+            return { path, sourceId, skipped: true, reason: "not-file" };
+        throw new Error(`Diary path is not a file: ${path}`);
+    }
+    if (options.store && isDiarySourceUnchanged(options.store, sourceId, path, fileStat)) {
+        return {
+            path,
+            sourceId,
+            result: { ids: [], embedded: 0, reused: 0, skipped: 0 },
+            skippedUnchanged: true,
+        };
+    }
+    const markdown = await readFile(path, "utf8");
+    const result = await indexDiaryMarkdown({
+        indexer: options.indexer,
+        path,
+        markdown,
+        signal: options.signal,
+    });
+    options.store?.upsertSourceState({
+        corpus: DIARY_CHUNK_CORPUS,
+        sourceId,
+        sourceRef: path,
+        mtimeMs: fileStat.mtimeMs,
+        sizeBytes: fileStat.size,
+    });
+    return { path, sourceId, result };
+}
+export async function removeDiaryFileIndex(options) {
+    const path = resolveDiaryPath(options.config, options.path);
+    const sourceId = basename(path);
+    const result = await options.indexer.replaceSource(DIARY_CHUNK_CORPUS, sourceId, [], options.signal);
+    return { path, sourceId, result };
+}
+export async function indexAllDiaryFiles(options) {
+    const paths = await listDiaryMarkdownFiles(options.config);
+    const files = [];
+    for (const path of paths) {
+        const result = await indexDiaryFile({ ...options, path });
+        if (!("skipped" in result))
+            files.push(result);
+    }
+    return { files };
+}
+function resolveDiaryPath(config, path) {
+    return isAbsolute(path) ? resolve(path) : resolve(config.memory.diariesDir, path);
+}
+function isDiarySourceUnchanged(store, sourceId, sourceRef, fileStat) {
+    const state = store.getSourceState(DIARY_CHUNK_CORPUS, sourceId);
+    if (!state?.hasMappings)
+        return false;
+    if (state.sourceRef !== sourceRef)
+        return false;
+    return state.mtimeMs === Math.floor(fileStat.mtimeMs) && state.sizeBytes === fileStat.size;
+}