opencode-mem 1.0.0

package/dist/services/compaction.js

@@ -0,0 +1,421 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { memoryClient } from "./client.js";
+import { log } from "./logger.js";
+import { CONFIG } from "../config.js";
+const MESSAGE_STORAGE = join(homedir(), ".opencode", "messages");
+const PART_STORAGE = join(homedir(), ".opencode", "parts");
+const DEFAULT_THRESHOLD = 0.80;
+const MIN_TOKENS_FOR_COMPACTION = 50_000;
+const COMPACTION_COOLDOWN_MS = 30_000;
+const CLAUDE_DEFAULT_CONTEXT_LIMIT = 200_000;
+const CLAUDE_MODEL_PATTERN = /claude-(opus|sonnet|haiku)/i;
+function createCompactionPrompt(projectMemories) {
+    const memoriesSection = projectMemories.length > 0
+        ? `
+## Project Knowledge (from Memory System)
+The following project-specific knowledge should be preserved and referenced in the summary:
+${projectMemories.map(m => `- ${m}`).join('\n')}
+`
+        : '';
+    return `[COMPACTION CONTEXT INJECTION]
+
+When summarizing this session, you MUST include the following sections in your summary:
+
+## 1. User Requests (As-Is)
+- List all original user requests exactly as they were stated
+- Preserve the user's exact wording and intent
+
+## 2. Final Goal
+- What the user ultimately wanted to achieve
+- The end result or deliverable expected
+
+## 3. Work Completed
+- What has been done so far
+- Files created/modified
+- Features implemented
+- Problems solved
+
+## 4. Remaining Tasks
+- What still needs to be done
+- Pending items from the original request
+- Follow-up tasks identified during the work
+
+## 5. MUST NOT Do (Critical Constraints)
+- Things that were explicitly forbidden
+- Approaches that failed and should not be retried
+- User's explicit restrictions or preferences
+- Anti-patterns identified during the session
+${memoriesSection}
+This context is critical for maintaining continuity after compaction.
+`;
+}
+function isSupportedModel(modelID) {
+    return CLAUDE_MODEL_PATTERN.test(modelID);
+}
+function getMessageDir(sessionID) {
+    if (!existsSync(MESSAGE_STORAGE))
+        return null;
+    const directPath = join(MESSAGE_STORAGE, sessionID);
+    if (existsSync(directPath))
+        return directPath;
+    for (const dir of readdirSync(MESSAGE_STORAGE)) {
+        const sessionPath = join(MESSAGE_STORAGE, dir, sessionID);
+        if (existsSync(sessionPath))
+            return sessionPath;
+    }
+    return null;
+}
+function getOrCreateMessageDir(sessionID) {
+    if (!existsSync(MESSAGE_STORAGE)) {
+        mkdirSync(MESSAGE_STORAGE, { recursive: true });
+    }
+    const directPath = join(MESSAGE_STORAGE, sessionID);
+    if (existsSync(directPath))
+        return directPath;
+    for (const dir of readdirSync(MESSAGE_STORAGE)) {
+        const sessionPath = join(MESSAGE_STORAGE, dir, sessionID);
+        if (existsSync(sessionPath))
+            return sessionPath;
+    }
+    mkdirSync(directPath, { recursive: true });
+    return directPath;
+}
+function findNearestMessageWithFields(messageDir) {
+    try {
+        const files = readdirSync(messageDir)
+            .filter((f) => f.endsWith(".json"))
+            .sort()
+            .reverse();
+        for (const file of files) {
+            try {
+                const content = readFileSync(join(messageDir, file), "utf-8");
+                const msg = JSON.parse(content);
+                if (msg.agent && msg.model?.providerID && msg.model?.modelID) {
+                    return msg;
+                }
+            }
+            catch {
+                continue;
+            }
+        }
+    }
+    catch {
+        return null;
+    }
+    return null;
+}
+function generateMessageId() {
+    const timestamp = Date.now().toString(16);
+    const random = Math.random().toString(36).substring(2, 14);
+    return `msg_${timestamp}${random}`;
+}
+function generatePartId() {
+    const timestamp = Date.now().toString(16);
+    const random = Math.random().toString(36).substring(2, 10);
+    return `prt_${timestamp}${random}`;
+}
+function injectHookMessage(sessionID, hookContent, originalMessage) {
+    if (!hookContent || hookContent.trim().length === 0) {
+        log("[compaction] attempted to inject empty content, skipping");
+        return false;
+    }
+    const messageDir = getOrCreateMessageDir(sessionID);
+    const fallback = findNearestMessageWithFields(messageDir);
+    const now = Date.now();
+    const messageID = generateMessageId();
+    const partID = generatePartId();
+    const resolvedAgent = originalMessage.agent ?? fallback?.agent ?? "general";
+    const resolvedModel = originalMessage.model?.providerID && originalMessage.model?.modelID
+        ? { providerID: originalMessage.model.providerID, modelID: originalMessage.model.modelID }
+        : fallback?.model?.providerID && fallback?.model?.modelID
+            ? { providerID: fallback.model.providerID, modelID: fallback.model.modelID }
+            : undefined;
+    const messageMeta = {
+        id: messageID,
+        sessionID,
+        role: "user",
+        time: { created: now },
+        agent: resolvedAgent,
+        model: resolvedModel,
+        path: originalMessage.path?.cwd
+            ? { cwd: originalMessage.path.cwd, root: originalMessage.path.root ?? "/" }
+            : undefined,
+    };
+    const textPart = {
+        id: partID,
+        type: "text",
+        text: hookContent,
+        synthetic: true,
+        time: { start: now, end: now },
+        messageID,
+        sessionID,
+    };
+    try {
+        writeFileSync(join(messageDir, `${messageID}.json`), JSON.stringify(messageMeta, null, 2));
+        const partDir = join(PART_STORAGE, messageID);
+        if (!existsSync(partDir)) {
+            mkdirSync(partDir, { recursive: true });
+        }
+        writeFileSync(join(partDir, `${partID}.json`), JSON.stringify(textPart, null, 2));
+        log("[compaction] hook message injected", { sessionID, messageID });
+        return true;
+    }
+    catch (err) {
+        log("[compaction] failed to inject hook message", { error: String(err) });
+        return false;
+    }
+}
+export function createCompactionHook(ctx, tags, options) {
+    const state = {
+        lastCompactionTime: new Map(),
+        compactionInProgress: new Set(),
+        summarizedSessions: new Set(),
+    };
+    const threshold = options?.threshold ?? DEFAULT_THRESHOLD;
+    const getModelLimit = options?.getModelLimit;
+    async function fetchProjectMemoriesForCompaction() {
+        try {
+            const result = await memoryClient.listMemories(tags.project, CONFIG.maxProjectMemories);
+            const memories = result.memories || [];
+            return memories.map((m) => m.summary || m.content || "").filter(Boolean);
+        }
+        catch (err) {
+            log("[compaction] failed to fetch project memories", { error: String(err) });
+            return [];
+        }
+    }
+    async function injectCompactionContext(summarizeCtx) {
+        log("[compaction] injecting context", { sessionID: summarizeCtx.sessionID });
+        const projectMemories = await fetchProjectMemoriesForCompaction();
+        const prompt = createCompactionPrompt(projectMemories);
+        const success = injectHookMessage(summarizeCtx.sessionID, prompt, {
+            agent: summarizeCtx.agent,
+            model: { providerID: summarizeCtx.providerID, modelID: summarizeCtx.modelID },
+            path: { cwd: summarizeCtx.directory },
+        });
+        if (success) {
+            log("[compaction] context injected with project memories", {
+                sessionID: summarizeCtx.sessionID,
+                memoriesCount: projectMemories.length
+            });
+        }
+    }
+    async function saveSummaryAsMemory(sessionID, summaryContent) {
+        if (!summaryContent || summaryContent.length < 100) {
+            log("[compaction] summary too short to save", { sessionID, length: summaryContent.length });
+            return;
+        }
+        try {
+            const result = await memoryClient.addMemory(`[Session Summary]\n${summaryContent}`, tags.project, { type: "conversation" });
+            if (result.success) {
+                log("[compaction] summary saved as memory", { sessionID, memoryId: result.id });
+            }
+            else {
+                log("[compaction] failed to save summary", { error: result.error });
+            }
+        }
+        catch (err) {
+            log("[compaction] failed to save summary", { error: String(err) });
+        }
+    }
+    async function checkAndTriggerCompaction(sessionID, lastAssistant) {
+        if (state.compactionInProgress.has(sessionID))
+            return;
+        const lastCompaction = state.lastCompactionTime.get(sessionID) ?? 0;
+        if (Date.now() - lastCompaction < COMPACTION_COOLDOWN_MS)
+            return;
+        if (lastAssistant.summary === true)
+            return;
+        const tokens = lastAssistant.tokens;
+        if (!tokens)
+            return;
+        let modelID = lastAssistant.modelID ?? "";
+        let providerID = lastAssistant.providerID ?? "";
+        let agent;
+        // Fallback: find model/agent from stored messages if not available
+        const messageDir = getMessageDir(sessionID);
+        const storedMessage = messageDir ? findNearestMessageWithFields(messageDir) : null;
+        if (!providerID || !modelID) {
+            if (storedMessage?.model?.providerID)
+                providerID = storedMessage.model.providerID;
+            if (storedMessage?.model?.modelID)
+                modelID = storedMessage.model.modelID;
+        }
+        agent = storedMessage?.agent;
+        if (!isSupportedModel(modelID)) {
+            log("[compaction] skipping unsupported model", { modelID });
+            return;
+        }
+        const configLimit = getModelLimit?.(providerID, modelID);
+        const contextLimit = configLimit ?? CLAUDE_DEFAULT_CONTEXT_LIMIT;
+        const totalUsed = tokens.input + tokens.cache.read + tokens.output;
+        if (totalUsed < MIN_TOKENS_FOR_COMPACTION)
+            return;
+        const usageRatio = totalUsed / contextLimit;
+        log("[compaction] checking", {
+            sessionID,
+            totalUsed,
+            contextLimit,
+            usageRatio: usageRatio.toFixed(2),
+            threshold,
+        });
+        if (usageRatio < threshold)
+            return;
+        state.compactionInProgress.add(sessionID);
+        state.lastCompactionTime.set(sessionID, Date.now());
+        if (!providerID || !modelID) {
+            state.compactionInProgress.delete(sessionID);
+            return;
+        }
+        await ctx.client.tui.showToast({
+            body: {
+                title: "Preemptive Compaction",
+                message: `Context at ${(usageRatio * 100).toFixed(0)}% - compacting with memory context...`,
+                variant: "warning",
+                duration: 3000,
+            },
+        }).catch(() => { });
+        log("[compaction] triggering compaction", { sessionID, usageRatio });
+        try {
+            await injectCompactionContext({
+                sessionID,
+                providerID,
+                modelID,
+                usageRatio,
+                directory: ctx.directory,
+                agent,
+            });
+            state.summarizedSessions.add(sessionID);
+            await ctx.client.session.summarize({
+                path: { id: sessionID },
+                body: { providerID, modelID },
+                query: { directory: ctx.directory },
+            });
+            await ctx.client.tui.showToast({
+                body: {
+                    title: "Compaction Complete",
+                    message: "Session compacted with memory context. Resuming...",
+                    variant: "success",
+                    duration: 2000,
+                },
+            }).catch(() => { });
+            state.compactionInProgress.delete(sessionID);
+            setTimeout(async () => {
+                try {
+                    const messageDir = getMessageDir(sessionID);
+                    const storedMessage = messageDir ? findNearestMessageWithFields(messageDir) : null;
+                    await ctx.client.session.promptAsync({
+                        path: { id: sessionID },
+                        body: {
+                            agent: storedMessage?.agent,
+                            parts: [{ type: "text", text: "Continue" }],
+                        },
+                        query: { directory: ctx.directory },
+                    });
+                }
+                catch { }
+            }, 500);
+        }
+        catch (err) {
+            log("[compaction] compaction failed", { sessionID, error: String(err) });
+            state.compactionInProgress.delete(sessionID);
+        }
+    }
+    async function handleSummaryMessage(sessionID, _messageInfo) {
+        log("[compaction] handleSummaryMessage called", { sessionID, inSet: state.summarizedSessions.has(sessionID) });
+        if (!state.summarizedSessions.has(sessionID))
+            return;
+        state.summarizedSessions.delete(sessionID);
+        log("[compaction] capturing summary for memory", { sessionID });
+        try {
+            const resp = await ctx.client.session.messages({
+                path: { id: sessionID },
+                query: { directory: ctx.directory },
+            });
+            const messages = (resp.data ?? resp);
+            const summaryMessage = messages.find(m => m.info.role === "assistant" &&
+                m.info.summary === true);
+            log("[compaction] looking for summary message", {
+                sessionID,
+                found: !!summaryMessage,
+                hasParts: !!summaryMessage?.parts
+            });
+            if (summaryMessage?.parts) {
+                const textParts = summaryMessage.parts.filter(p => p.type === "text" && p.text);
+                const summaryContent = textParts.map(p => p.text).join("\n");
+                log("[compaction] summary content", {
+                    sessionID,
+                    textPartsCount: textParts.length,
+                    contentLength: summaryContent.length
+                });
+                if (summaryContent) {
+                    await saveSummaryAsMemory(sessionID, summaryContent);
+                }
+            }
+        }
+        catch (err) {
+            log("[compaction] failed to capture summary", { error: String(err) });
+        }
+    }
+    return {
+        compactionTracker: state.lastCompactionTime,
+        async event({ event }) {
+            const props = event.properties;
+            if (event.type === "session.deleted") {
+                const sessionInfo = props?.info;
+                if (sessionInfo?.id) {
+                    state.lastCompactionTime.delete(sessionInfo.id);
+                    state.compactionInProgress.delete(sessionInfo.id);
+                    state.summarizedSessions.delete(sessionInfo.id);
+                }
+                return;
+            }
+            if (event.type === "message.updated") {
+                const info = props?.info;
+                if (!info)
+                    return;
+                const sessionID = info.sessionID;
+                if (!sessionID)
+                    return;
+                if (info.role === "assistant" && info.summary === true && info.finish) {
+                    await handleSummaryMessage(sessionID, info);
+                    return;
+                }
+                if (info.role !== "assistant" || !info.finish)
+                    return;
+                await checkAndTriggerCompaction(sessionID, info);
+                return;
+            }
+            if (event.type === "session.idle") {
+                const sessionID = props?.sessionID;
+                if (!sessionID)
+                    return;
+                try {
+                    const resp = await ctx.client.session.messages({
+                        path: { id: sessionID },
+                        query: { directory: ctx.directory },
+                    });
+                    const messages = (resp.data ?? resp);
+                    const assistants = messages
+                        .filter((m) => m.info.role === "assistant")
+                        .map((m) => m.info);
+                    if (assistants.length === 0)
+                        return;
+                    const lastAssistant = assistants[assistants.length - 1];
+                    if (!lastAssistant.providerID || !lastAssistant.modelID) {
+                        const messageDir = getMessageDir(sessionID);
+                        const storedMessage = messageDir ? findNearestMessageWithFields(messageDir) : null;
+                        if (storedMessage?.model?.providerID && storedMessage?.model?.modelID) {
+                            lastAssistant.providerID = storedMessage.model.providerID;
+                            lastAssistant.modelID = storedMessage.model.modelID;
+                        }
+                    }
+                    await checkAndTriggerCompaction(sessionID, lastAssistant);
+                }
+                catch { }
+            }
+        },
+    };
+}

package/dist/services/context.d.ts

@@ -0,0 +1,17 @@
+interface MemoryResultMinimal {
+    similarity: number;
+    memory?: string;
+    chunk?: string;
+}
+interface MemoriesResponseMinimal {
+    results?: MemoryResultMinimal[];
+}
+interface ProfileResponse {
+    profile?: {
+        static: string[];
+        dynamic: string[];
+    };
+}
+export declare function formatContextForPrompt(profile: ProfileResponse | null, userMemories: MemoriesResponseMinimal, projectMemories: MemoriesResponseMinimal): string;
+export {};
+//# sourceMappingURL=context.d.ts.map

package/dist/services/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/services/context.ts"],"names":[],"mappings":"AAEA,UAAU,mBAAmB;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,uBAAuB;IAC/B,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE;QACR,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;CACH;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,eAAe,GAAG,IAAI,EAC/B,YAAY,EAAE,uBAAuB,EACrC,eAAe,EAAE,uBAAuB,GACvC,MAAM,CA8CR"}

package/dist/services/context.js

@@ -0,0 +1,41 @@
+import { CONFIG } from "../config.js";
+export function formatContextForPrompt(profile, userMemories, projectMemories) {
+    const parts = ["[MEMORY]"];
+    if (CONFIG.injectProfile && profile?.profile) {
+        const { static: staticFacts, dynamic: dynamicFacts } = profile.profile;
+        if (staticFacts.length > 0) {
+            parts.push("\nUser Profile:");
+            staticFacts.slice(0, CONFIG.maxProfileItems).forEach((fact) => {
+                parts.push(`- ${fact}`);
+            });
+        }
+        if (dynamicFacts.length > 0) {
+            parts.push("\nRecent Context:");
+            dynamicFacts.slice(0, CONFIG.maxProfileItems).forEach((fact) => {
+                parts.push(`- ${fact}`);
+            });
+        }
+    }
+    const projectResults = projectMemories.results || [];
+    if (projectResults.length > 0) {
+        parts.push("\nProject Knowledge:");
+        projectResults.forEach((mem) => {
+            const similarity = Math.round(mem.similarity * 100);
+            const content = mem.memory || mem.chunk || "";
+            parts.push(`- [${similarity}%] ${content}`);
+        });
+    }
+    const userResults = userMemories.results || [];
+    if (userResults.length > 0) {
+        parts.push("\nRelevant Memories:");
+        userResults.forEach((mem) => {
+            const similarity = Math.round(mem.similarity * 100);
+            const content = mem.memory || mem.chunk || "";
+            parts.push(`- [${similarity}%] ${content}`);
+        });
+    }
+    if (parts.length === 1) {
+        return "";
+    }
+    return parts.join("\n");
+}

package/dist/services/deduplication-service.d.ts

@@ -0,0 +1,30 @@
+interface DuplicateGroup {
+    representative: {
+        id: string;
+        content: string;
+        containerTag: string;
+        createdAt: number;
+    };
+    duplicates: Array<{
+        id: string;
+        content: string;
+        similarity: number;
+    }>;
+}
+interface DeduplicationResult {
+    exactDuplicatesDeleted: number;
+    nearDuplicateGroups: DuplicateGroup[];
+}
+export declare class DeduplicationService {
+    private isRunning;
+    detectAndRemoveDuplicates(): Promise<DeduplicationResult>;
+    private cosineSimilarity;
+    getStatus(): {
+        enabled: boolean;
+        threshold: number;
+        isRunning: boolean;
+    };
+}
+export declare const deduplicationService: DeduplicationService;
+export {};
+//# sourceMappingURL=deduplication-service.d.ts.map

package/dist/services/deduplication-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplication-service.d.ts","sourceRoot":"","sources":["../../src/services/deduplication-service.ts"],"names":[],"mappings":"AAOA,UAAU,cAAc;IACtB,cAAc,EAAE;QACd,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;CACJ;AAED,UAAU,mBAAmB;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,cAAc,EAAE,CAAC;CACvC;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,SAAS,CAAkB;IAE7B,yBAAyB,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAgH/D,OAAO,CAAC,gBAAgB;IAoBxB,SAAS;;;;;CAOV;AAED,eAAO,MAAM,oBAAoB,sBAA6B,CAAC"}

package/dist/services/deduplication-service.js

@@ -0,0 +1,131 @@
+import { embeddingService } from "./embedding.js";
+import { shardManager } from "./sqlite/shard-manager.js";
+import { vectorSearch } from "./sqlite/vector-search.js";
+import { connectionManager } from "./sqlite/connection-manager.js";
+import { CONFIG } from "../config.js";
+import { log } from "./logger.js";
+export class DeduplicationService {
+    isRunning = false;
+    async detectAndRemoveDuplicates() {
+        if (this.isRunning) {
+            throw new Error("Deduplication already running");
+        }
+        if (!CONFIG.deduplicationEnabled) {
+            throw new Error("Deduplication is disabled in config");
+        }
+        this.isRunning = true;
+        try {
+            log("Deduplication: starting", {
+                threshold: CONFIG.deduplicationSimilarityThreshold,
+            });
+            const userShards = shardManager.getAllShards("user", "");
+            const projectShards = shardManager.getAllShards("project", "");
+            const allShards = [...userShards, ...projectShards];
+            let exactDeleted = 0;
+            const nearDuplicateGroups = [];
+            for (const shard of allShards) {
+                const db = connectionManager.getConnection(shard.dbPath);
+                const memories = vectorSearch.getAllMemories(db);
+                const contentMap = new Map();
+                for (const memory of memories) {
+                    const key = `${memory.container_tag}:${memory.content}`;
+                    if (!contentMap.has(key)) {
+                        contentMap.set(key, []);
+                    }
+                    contentMap.get(key).push(memory);
+                }
+                for (const [key, duplicates] of contentMap) {
+                    if (duplicates.length > 1) {
+                        duplicates.sort((a, b) => Number(b.created_at) - Number(a.created_at));
+                        const keep = duplicates[0];
+                        const toDelete = duplicates.slice(1);
+                        for (const dup of toDelete) {
+                            try {
+                                vectorSearch.deleteVector(db, dup.id);
+                                shardManager.decrementVectorCount(shard.id);
+                                exactDeleted++;
+                            }
+                            catch (error) {
+                                log("Deduplication: delete error", {
+                                    memoryId: dup.id,
+                                    error: String(error),
+                                });
+                            }
+                        }
+                    }
+                }
+                const uniqueMemories = Array.from(contentMap.values()).map((arr) => arr[0]);
+                for (let i = 0; i < uniqueMemories.length; i++) {
+                    const mem1 = uniqueMemories[i];
+                    if (!mem1.vector)
+                        continue;
+                    const vector1 = new Float32Array(new Uint8Array(mem1.vector).buffer);
+                    const similarGroup = {
+                        representative: {
+                            id: mem1.id,
+                            content: mem1.content,
+                            containerTag: mem1.container_tag,
+                            createdAt: mem1.created_at,
+                        },
+                        duplicates: [],
+                    };
+                    for (let j = i + 1; j < uniqueMemories.length; j++) {
+                        const mem2 = uniqueMemories[j];
+                        if (!mem2.vector)
+                            continue;
+                        if (mem1.container_tag !== mem2.container_tag)
+                            continue;
+                        const vector2 = new Float32Array(new Uint8Array(mem2.vector).buffer);
+                        const similarity = this.cosineSimilarity(vector1, vector2);
+                        if (similarity >= CONFIG.deduplicationSimilarityThreshold && similarity < 1.0) {
+                            similarGroup.duplicates.push({
+                                id: mem2.id,
+                                content: mem2.content,
+                                similarity,
+                            });
+                        }
+                    }
+                    if (similarGroup.duplicates.length > 0) {
+                        nearDuplicateGroups.push(similarGroup);
+                    }
+                }
+            }
+            log("Deduplication: completed", {
+                exactDeleted,
+                nearDuplicateGroupsFound: nearDuplicateGroups.length,
+            });
+            return {
+                exactDuplicatesDeleted: exactDeleted,
+                nearDuplicateGroups,
+            };
+        }
+        finally {
+            this.isRunning = false;
+        }
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length)
+            return 0;
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            const aVal = a[i] || 0;
+            const bVal = b[i] || 0;
+            dotProduct += aVal * bVal;
+            normA += aVal * aVal;
+            normB += bVal * bVal;
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    getStatus() {
+        return {
+            enabled: CONFIG.deduplicationEnabled,
+            threshold: CONFIG.deduplicationSimilarityThreshold,
+            isRunning: this.isRunning,
+        };
+    }
+}
+export const deduplicationService = new DeduplicationService();

package/dist/services/embedding.d.ts

@@ -0,0 +1,10 @@
+export declare class EmbeddingService {
+    private pipe;
+    private initPromise;
+    isWarmedUp: boolean;
+    warmup(progressCallback?: (progress: any) => void): Promise<void>;
+    embed(text: string): Promise<Float32Array>;
+    embedWithTimeout(text: string): Promise<Float32Array>;
+}
+export declare const embeddingService: EmbeddingService;
+//# sourceMappingURL=embedding.d.ts.map

package/dist/services/embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/services/embedding.ts"],"names":[],"mappings":"AAiBA,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,WAAW,CAA8B;IAC1C,UAAU,EAAE,OAAO,CAAS;IAE7B,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BjE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAkC1C,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;CAG5D;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}