@qearlyao/familiar 0.1.0

package/dist/inbound-attachments.js

@@ -0,0 +1,266 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
+import { basename, extname, resolve } from "node:path";
+import { attachmentsDir, publicAttachmentPath } from "./generated-media.js";
+import { ensureInlineImageDerivative, MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
+import { deriveInboundAttachmentText } from "./media-understanding.js";
+export { MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
+export const MAX_INBOUND_ATTACHMENTS = 4;
+export const MAX_INBOUND_ATTACHMENT_BYTES = 12 * 1024 * 1024;
+export const MAX_INBOUND_TOTAL_BYTES = 24 * 1024 * 1024;
+const ALLOWED_MIME_TYPES = new Set([
+    "image/jpeg",
+    "image/png",
+    "image/gif",
+    "image/webp",
+    "audio/mpeg",
+    "audio/ogg",
+    "audio/wav",
+    "audio/webm",
+    "video/mp4",
+    "video/webm",
+    "application/pdf",
+    "text/plain",
+]);
+const EXTENSIONS_BY_MIME = {
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/gif": ".gif",
+    "image/webp": ".webp",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "audio/wav": ".wav",
+    "audio/webm": ".webm",
+    "video/mp4": ".mp4",
+    "video/webm": ".webm",
+    "application/pdf": ".pdf",
+    "text/plain": ".txt",
+};
+function safeName(name, fallback) {
+    const base = basename(name || fallback)
+        .replace(/[^A-Za-z0-9._=-]+/g, "_")
+        .slice(0, 120);
+    return base || fallback;
+}
+function kindFromMime(mimeType) {
+    if (mimeType.startsWith("image/"))
+        return "image";
+    if (mimeType.startsWith("audio/"))
+        return "audio";
+    if (mimeType.startsWith("video/"))
+        return "video";
+    return "file";
+}
+function sniffText(buffer) {
+    if (buffer.length === 0)
+        return "text/plain";
+    const head = buffer.subarray(0, Math.min(buffer.length, 512));
+    if (head.includes(0))
+        return undefined;
+    return head.every((byte) => byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126))
+        ? "text/plain"
+        : undefined;
+}
+function sniffMimeType(buffer, declared) {
+    let detected;
+    if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
+        detected = "image/jpeg";
+    else if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
+        detected = "image/png";
+    }
+    else if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" ||
+        buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
+        detected = "image/gif";
+    }
+    else if (buffer.subarray(0, 4).toString("ascii") === "RIFF" &&
+        buffer.subarray(8, 12).toString("ascii") === "WEBP") {
+        detected = "image/webp";
+    }
+    else if (buffer.subarray(0, 4).toString("ascii") === "%PDF")
+        detected = "application/pdf";
+    else if (buffer.subarray(0, 3).toString("ascii") === "ID3" ||
+        buffer.subarray(0, 2).equals(Buffer.from([0xff, 0xfb]))) {
+        detected = "audio/mpeg";
+    }
+    else if (buffer.subarray(0, 4).toString("ascii") === "OggS")
+        detected = "audio/ogg";
+    else if (buffer.subarray(0, 4).toString("ascii") === "RIFF" && buffer.subarray(8, 12).toString("ascii") === "WAVE") {
+        detected = "audio/wav";
+    }
+    const mime = detected ?? sniffText(buffer) ?? declared;
+    if (!mime || !ALLOWED_MIME_TYPES.has(mime)) {
+        throw new Error(`Unsupported attachment type: ${declared || detected || "unknown"}`);
+    }
+    return mime;
+}
+function canonicalName(name, fallbackStem, mimeType) {
+    const current = safeName(name, fallbackStem);
+    const stem = current.replace(/\.[^.]+$/, "");
+    return `${stem}${EXTENSIONS_BY_MIME[mimeType] || extname(current) || ""}`;
+}
+async function fetchRemoteAttachment(input, signal) {
+    if (!input.url)
+        throw new Error("Attachment URL is required");
+    const response = await fetch(input.url, { signal });
+    if (!response.ok)
+        throw new Error(`Attachment download failed: HTTP ${response.status}`);
+    const contentLength = Number(response.headers.get("content-length") ?? 0);
+    if (contentLength > MAX_INBOUND_ATTACHMENT_BYTES) {
+        throw new Error(`Attachment is too large: ${contentLength} bytes`);
+    }
+    const reader = response.body?.getReader();
+    if (!reader) {
+        const arrayBuffer = await response.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        if (buffer.byteLength > MAX_INBOUND_ATTACHMENT_BYTES) {
+            throw new Error(`Attachment is too large: ${buffer.byteLength} bytes`);
+        }
+        return buffer;
+    }
+    const chunks = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            const chunk = Buffer.from(value);
+            total += chunk.byteLength;
+            if (total > MAX_INBOUND_ATTACHMENT_BYTES) {
+                throw new Error(`Attachment is too large: ${total} bytes`);
+            }
+            chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+async function attachmentBuffer(input) {
+    if (input.buffer) {
+        if (input.buffer.byteLength > MAX_INBOUND_ATTACHMENT_BYTES) {
+            throw new Error(`Attachment is too large: ${input.buffer.byteLength} bytes`);
+        }
+        return input.buffer;
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 15_000);
+    try {
+        return await fetchRemoteAttachment(input, controller.signal);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function materializeInboundAttachments(config, inputs) {
+    if (inputs.length > MAX_INBOUND_ATTACHMENTS) {
+        throw new Error(`Too many attachments: max ${MAX_INBOUND_ATTACHMENTS}`);
+    }
+    const prepared = [];
+    let totalBytes = 0;
+    for (const [index, input] of inputs.entries()) {
+        const buffer = await attachmentBuffer(input);
+        totalBytes += buffer.byteLength;
+        if (totalBytes > MAX_INBOUND_TOTAL_BYTES) {
+            throw new Error(`Attachments are too large: max ${MAX_INBOUND_TOTAL_BYTES} bytes total`);
+        }
+        const mimeType = sniffMimeType(buffer, input.mimeType);
+        const id = input.id || randomUUID();
+        const cleanName = canonicalName(input.name, `attachment-${index + 1}`, mimeType);
+        const sha256 = createHash("sha256").update(buffer).digest("hex");
+        prepared.push({
+            id,
+            name: cleanName,
+            kind: kindFromMime(mimeType),
+            mimeType,
+            size: buffer.byteLength,
+            sha256,
+            buffer,
+            remoteUrl: input.url,
+            sourceUrl: input.url,
+            source: input.source,
+        });
+    }
+    const stored = [];
+    const writtenPaths = [];
+    try {
+        for (const attachment of prepared) {
+            const dir = resolve(attachmentsDir(config), "inbound", attachment.source);
+            await mkdir(dir, { recursive: true });
+            const localPath = resolve(dir, `${Date.now()}-${attachment.id}-${attachment.name}`);
+            await writeFile(localPath, attachment.buffer);
+            writtenPaths.push(localPath);
+            const finalAttachment = {
+                id: attachment.id,
+                name: attachment.name,
+                kind: attachment.kind,
+                mimeType: attachment.mimeType,
+                size: attachment.size,
+                remoteUrl: attachment.remoteUrl,
+                sourceUrl: attachment.sourceUrl,
+                localPath,
+                source: attachment.source,
+                sha256: attachment.sha256,
+            };
+            const derivedImage = await ensureInlineImageDerivative(config, finalAttachment);
+            if (derivedImage) {
+                finalAttachment.derived = {
+                    ...finalAttachment.derived,
+                    image: derivedImage,
+                };
+            }
+            stored.push(finalAttachment);
+        }
+        return await deriveInboundAttachmentText(config, stored);
+    }
+    catch (error) {
+        await Promise.all(writtenPaths.map((path) => unlink(path).catch(() => undefined)));
+        throw error;
+    }
+}
+export async function promptImagesFromAttachments(attachments) {
+    const images = [];
+    const notes = [];
+    for (const attachment of attachments) {
+        if (!attachment.localPath || !attachment.mimeType?.startsWith("image/"))
+            continue;
+        if (attachment.kind && attachment.kind !== "image")
+            continue;
+        const imageMeta = attachment.derived?.image;
+        const imagePath = imageMeta?.localPath || attachment.localPath;
+        const data = (await readFile(imagePath)).toString("base64");
+        if (Buffer.byteLength(data, "utf8") > MAX_INLINE_IMAGE_BASE64_BYTES) {
+            notes.push(`<attachment name="${attachment.name}" mime="${attachment.mimeType}">[Image omitted from model input: inline payload is too large.]</attachment>`);
+            continue;
+        }
+        images.push({
+            type: "image",
+            mimeType: imageMeta?.mimeType ?? attachment.mimeType,
+            data,
+        });
+        const detail = imageMeta?.note ? ` ${imageMeta.note}` : "";
+        notes.push(`<attachment name="${attachment.name}" mime="${attachment.mimeType}">${detail}</attachment>`);
+    }
+    return {
+        images,
+        promptSuffix: notes.join("\n"),
+    };
+}
+export function promptAttachmentNotes(attachments) {
+    return attachments
+        .map((attachment) => {
+        const attrs = `name="${attachment.name}" id="${attachment.id}" kind="${attachment.kind ?? "file"}" mime="${attachment.mimeType ?? "unknown"}" size="${attachment.size ?? "unknown"}"`;
+        const derivedText = attachment.derived?.text?.text;
+        if (derivedText) {
+            const label = attachment.derived?.text?.label || (attachment.kind === "audio" ? "transcription" : "summary");
+            return `<attachment ${attrs}>[${label}: ${derivedText}]</attachment>`;
+        }
+        return `<attachment ${attrs}></attachment>`;
+    })
+        .join("\n")
+        .trim();
+}
+export function publicInboundAttachmentPath(config, localPath) {
+    return publicAttachmentPath(config, localPath);
+}

package/dist/index.js

@@ -0,0 +1,10 @@
+export { createFamiliarAgent } from "./agent.js";
+export { buildRecordBase, chatChannelKey, chatLogPath, createChatLog, } from "./chat-log.js";
+export { loadConfig } from "./config.js";
+export { startDiscordDaemon } from "./discord.js";
+export { clampConfiguredThinkingLevel, createConfiguredModel, describeModelAuth, formatAllowedModels, isAllowedModel, isThinkingLevel, parseModelRef, resolveModel, resolveModelApiKey, supportedThinkingLevels, } from "./models.js";
+export { buildSystemPrompt, loadPersona } from "./persona.js";
+export { ConversationRuntime, } from "./runtime.js";
+export { loadSettingsStore, } from "./settings.js";
+export { formatFamiliarSkillsForPrompt, loadFamiliarSkills } from "./skills.js";
+export { startWebDaemon } from "./web.js";

package/dist/media-understanding.js

@@ -0,0 +1,120 @@
+import { readFile } from "node:fs/promises";
+import { createPartFromBase64, createUserContent, GoogleGenAI } from "@google/genai";
+import { parseModelRef, resolveModel } from "./models.js";
+const GEMINI_API_VERSION_PATTERN = /\/(v1(?:beta|alpha)?|v\d+beta\d*)\/?$/;
+function normalizeDerivedText(text) {
+    return text.trim().replace(/\n{3,}/g, "\n\n");
+}
+function labelForAttachment(kind) {
+    if (kind === "audio")
+        return "transcription";
+    if (kind === "video")
+        return "summary";
+    return "text";
+}
+function geminiHttpOptions(config) {
+    const ref = parseModelRef(`google/${config.mediaUnderstanding.video.model}`);
+    const model = ref ? resolveModel(ref, config) : undefined;
+    const baseUrl = model?.baseUrl;
+    if (!baseUrl)
+        return { timeout: 60_000 };
+    const match = baseUrl.match(GEMINI_API_VERSION_PATTERN);
+    if (!match)
+        return { baseUrl, timeout: 60_000 };
+    return {
+        baseUrl: baseUrl.slice(0, match.index),
+        apiVersion: match[1],
+        timeout: 60_000,
+    };
+}
+async function transcribeAudioAttachment(config, attachment) {
+    if (!attachment.localPath || !attachment.mimeType?.startsWith("audio/"))
+        return undefined;
+    const apiKey = process.env[config.mediaUnderstanding.audio.apiKeyEnv];
+    if (!apiKey) {
+        console.warn(`media understanding skipped: ${config.mediaUnderstanding.audio.apiKeyEnv} is not set`);
+        return undefined;
+    }
+    const form = new FormData();
+    form.set("model", config.mediaUnderstanding.audio.model);
+    form.set("file", new Blob([await readFile(attachment.localPath)], { type: attachment.mimeType }), attachment.name);
+    const response = await fetch("https://api.groq.com/openai/v1/audio/transcriptions", {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: form,
+        signal: AbortSignal.timeout(30_000),
+    });
+    if (!response.ok)
+        throw new Error(`Groq transcription failed: HTTP ${response.status}`);
+    const parsed = (await response.json());
+    const text = parsed.text?.trim();
+    if (!text)
+        return undefined;
+    return {
+        provider: "groq",
+        model: config.mediaUnderstanding.audio.model,
+        text: normalizeDerivedText(text),
+        label: labelForAttachment(attachment.kind),
+    };
+}
+async function summarizeVideoAttachment(config, attachment) {
+    if (!attachment.localPath || !attachment.mimeType?.startsWith("video/"))
+        return undefined;
+    const apiKey = process.env[config.mediaUnderstanding.video.apiKeyEnv];
+    if (!apiKey) {
+        console.warn(`media understanding skipped: ${config.mediaUnderstanding.video.apiKeyEnv} is not set`);
+        return undefined;
+    }
+    const ai = new GoogleGenAI({ apiKey, httpOptions: geminiHttpOptions(config) });
+    const video = await readFile(attachment.localPath);
+    const response = await ai.models.generateContent({
+        model: config.mediaUnderstanding.video.model,
+        contents: createUserContent([
+            {
+                text: "Provide a concise description of this video, including any spoken content if present, and summarize the key visible events.",
+            },
+            createPartFromBase64(video.toString("base64"), attachment.mimeType),
+        ]),
+    });
+    const text = response.text?.trim();
+    if (!text)
+        return undefined;
+    return {
+        provider: "google",
+        model: config.mediaUnderstanding.video.model,
+        text: normalizeDerivedText(text),
+        label: labelForAttachment(attachment.kind),
+    };
+}
+export async function deriveInboundAttachmentText(config, attachments) {
+    const next = [];
+    for (const attachment of attachments) {
+        if (attachment.derived?.text || !attachment.localPath) {
+            next.push(attachment);
+            continue;
+        }
+        try {
+            if (attachment.mimeType?.startsWith("audio/")) {
+                const text = await transcribeAudioAttachment(config, attachment);
+                if (text) {
+                    next.push({ ...attachment, derived: { ...(attachment.derived ?? {}), text } });
+                    continue;
+                }
+            }
+            if (attachment.mimeType?.startsWith("video/")) {
+                const text = await summarizeVideoAttachment(config, attachment);
+                if (text) {
+                    next.push({ ...attachment, derived: { ...(attachment.derived ?? {}), text } });
+                    continue;
+                }
+            }
+        }
+        catch (error) {
+            console.error("media understanding failed", error);
+        }
+        next.push(attachment);
+    }
+    return next;
+}

package/dist/memory/diary/ambient-injector.js

@@ -0,0 +1,180 @@
+import { retrieveAmbientDiary } from "./ambient.js";
+const INJECTED_MEMORY_OPEN = "<injected_memory>";
+const INJECTED_MEMORY_CLOSE = "</injected_memory>";
+const INJECTED_MEMORY_BLOCK_RE = /<injected_memory\b[^>]*>[\s\S]*?<\/injected_memory>/gi;
+export class AmbientDiaryInjector {
+    enabled;
+    store;
+    embeddingProvider;
+    topK;
+    minQueryLength;
+    throttleMs;
+    weightSimilarity;
+    weightValence;
+    weightRecency;
+    weightIntensity;
+    now;
+    lastInjectedAtBySession = new Map();
+    constructor(options) {
+        this.enabled = options.enabled ?? true;
+        this.store = options.store;
+        this.embeddingProvider = options.embeddingProvider;
+        this.topK = positiveIntegerOrDefault(options.topK, 3);
+        this.minQueryLength = nonNegativeIntegerOrDefault(options.minQueryLength, 8);
+        this.throttleMs = nonNegativeIntegerOrDefault(options.throttleSeconds, 30) * 1000;
+        this.weightSimilarity = nonNegativeNumberOrDefault(options.weightSimilarity, 1.0);
+        this.weightValence = nonNegativeNumberOrDefault(options.weightValence, 0.08);
+        this.weightRecency = nonNegativeNumberOrDefault(options.weightRecency, 0.08);
+        this.weightIntensity = nonNegativeNumberOrDefault(options.weightIntensity, 0.1);
+        this.now = options.now ?? Date.now;
+    }
+    async inject(messages, signal, sessionKey = "default") {
+        if (!this.enabled)
+            return messages;
+        try {
+            const query = lastUserText(messages);
+            if (!query || query.length < this.minQueryLength)
+                return messages;
+            const now = this.now();
+            const lastInjectedAt = this.lastInjectedAtBySession.get(sessionKey);
+            if (lastInjectedAt !== undefined && this.throttleMs > 0 && now - lastInjectedAt < this.throttleMs)
+                return messages;
+            debugAmbientQuery(sessionKey, query);
+            const hits = await retrieveAmbientDiary({
+                query,
+                store: this.store,
+                embeddingProvider: this.embeddingProvider,
+                limit: this.topK,
+                weights: {
+                    similarity: this.weightSimilarity,
+                    valence: this.weightValence,
+                    recency: this.weightRecency,
+                    intensity: this.weightIntensity,
+                },
+                signal,
+            });
+            if (hits.length === 0)
+                return messages;
+            this.lastInjectedAtBySession.set(sessionKey, now);
+            return injectAmbientDiaryRecall(messages, renderAmbientDiaryRecall(hits));
+        }
+        catch (error) {
+            console.error("memory ambient recall failed", error);
+            return messages;
+        }
+    }
+}
+function positiveIntegerOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
+}
+function nonNegativeIntegerOrDefault(value, fallback) {
+    return value !== undefined && Number.isInteger(value) && value >= 0 ? value : fallback;
+}
+function nonNegativeNumberOrDefault(value, fallback) {
+    return value !== undefined && Number.isFinite(value) && value >= 0 ? value : fallback;
+}
+function injectAmbientDiaryRecall(messages, recallText) {
+    const lastUserIndex = findLastUserMessageIndex(messages);
+    if (lastUserIndex < 0)
+        return messages;
+    return messages.map((message, index) => index === lastUserIndex ? appendTextToUserMessage(message, `\n\n${recallText}`) : message);
+}
+function appendTextToUserMessage(message, text) {
+    if (message.role !== "user")
+        return message;
+    if (typeof message.content === "string")
+        return { ...message, content: `${message.content}${text}` };
+    return {
+        ...message,
+        content: [...message.content, { type: "text", text }],
+    };
+}
+function findLastUserMessageIndex(messages) {
+    for (let index = messages.length - 1; index >= 0; index--) {
+        if (messages[index]?.role === "user")
+            return index;
+    }
+    return -1;
+}
+function lastUserText(messages) {
+    const index = findLastUserMessageIndex(messages);
+    if (index < 0)
+        return "";
+    const message = messages[index];
+    if (!message || message.role !== "user")
+        return "";
+    const text = typeof message.content === "string"
+        ? message.content
+        : message.content
+            .filter(isTextPart)
+            .map((item) => item.text)
+            .join("\n");
+    return stripInjectedMemoryBlocks(text).trim();
+}
+function isTextPart(item) {
+    return item.type === "text" && typeof item.text === "string";
+}
+function stripInjectedMemoryBlocks(text) {
+    return text.replace(INJECTED_MEMORY_BLOCK_RE, "").trim();
+}
+function debugAmbientQuery(sessionKey, query) {
+    if (!process.env.DEBUG?.split(",")
+        .map((part) => part.trim())
+        .includes("memory-ambient"))
+        return;
+    console.error(JSON.stringify({ event: "ambient_diary_query", sessionKey, query }));
+}
+function renderAmbientDiaryRecall(hits) {
+    const lines = [INJECTED_MEMORY_OPEN];
+    for (const [index, hit] of hits.entries()) {
+        const date = typeof hit.chunk.metadata?.date === "string" ? hit.chunk.metadata.date : undefined;
+        const heading = typeof hit.chunk.metadata?.heading === "string" ? hit.chunk.metadata.heading : undefined;
+        const rawLabel = [date, heading].filter(Boolean).join(" ");
+        const label = diaryLabel(date, heading);
+        const prefix = label ? `${index + 1}. ${label}` : `${index + 1}. diary`;
+        const text = stripRepeatedDiaryPrefix(hit.chunk.snippet || hit.chunk.text, [rawLabel, label]);
+        lines.push(`${escapeXmlText(prefix)}: ${escapeXmlText(text)}`);
+    }
+    lines.push(INJECTED_MEMORY_CLOSE);
+    return lines.join("\n");
+}
+function diaryLabel(date, heading) {
+    return uniqueNonEmptyStrings([date, heading]).join(" ");
+}
+function stripRepeatedDiaryPrefix(text, labels) {
+    let trimmed = text.trim();
+    for (const label of uniqueNonEmptyStrings(labels)) {
+        const escaped = escapeRegExp(label);
+        trimmed = trimmed.replace(new RegExp(`^(?:${escaped}\\s*:\\s*)+`, "i"), "").trim();
+    }
+    return trimmed;
+}
+function uniqueNonEmptyStrings(values) {
+    const out = [];
+    const seen = new Set();
+    for (const value of values) {
+        const normalized = value?.trim();
+        if (!normalized)
+            continue;
+        const key = normalized.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(normalized);
+    }
+    return out;
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function escapeXmlText(value) {
+    return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+export const __ambientDiaryInjectorTest = {
+    injectAmbientDiaryRecall,
+    lastUserText,
+    renderAmbientDiaryRecall,
+    diaryLabel,
+    stripInjectedMemoryBlocks,
+    stripRepeatedDiaryPrefix,
+};