@lotargo/memory_plugin 1.0.3 → 1.1.1

package/mcp-server/ingest/sentence_segmenter.js

@@ -0,0 +1,74 @@
+// Multilingual Sentence Segmenter using Intl.Segmenter with Abbreviation & Boundary Protection
+
+const RU_ABBREVIATIONS = new Set([
+  "т.д", "т.п", "т.е", "и др", "г", "гг", "ул", "руб", "коп", "стр", "рис", "см", "им", "пер", "д", "к", "п", "в"
+]);
+
+const EN_ABBREVIATIONS = new Set([
+  "e.g", "i.e", "etc", "vs", "dr", "mr", "mrs", "ms", "prof", "inc", "ltd", "co", "vol", "no", "p", "pp", "fig"
+]);
+
+function isAbbreviation(word) {
+  if (!word) return false;
+  const clean = word.trim().toLowerCase().replace(/\.$/, "");
+  return RU_ABBREVIATIONS.has(clean) || EN_ABBREVIATIONS.has(clean);
+}
+
+export function splitSentencesMultilingual(text, langHint = "ru") {
+  if (!text || typeof text !== "string") return [];
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return [];
+
+  let rawSegments = [];
+
+  if (typeof Intl !== "undefined" && Intl.Segmenter) {
+    try {
+      const segmenter = new Intl.Segmenter([langHint, "ru", "en"], { granularity: "sentence" });
+      const iter = segmenter.segment(trimmed);
+      for (const seg of iter) {
+        if (seg.segment.trim().length > 0) {
+          rawSegments.push(seg.segment);
+        }
+      }
+    } catch {
+      rawSegments = fallbackSentenceSplit(trimmed);
+    }
+  } else {
+    rawSegments = fallbackSentenceSplit(trimmed);
+  }
+
+  // Post-process & merge false splits caused by abbreviations or numbers (e.g., "v1.0", "e.g.", "т. д.")
+  const sentences = [];
+  let current = "";
+
+  for (let i = 0; i < rawSegments.length; i++) {
+    const seg = rawSegments[i];
+    if (!current) {
+      current = seg;
+    } else {
+      const lastWordMatch = current.trim().match(/([a-zA-Zа-яА-Я0-9._-]+)\s*\.?$/);
+      const lastWord = lastWordMatch ? lastWordMatch[1] : "";
+      
+      const isNumDot = /\b\d+\.$/.test(current.trim());
+      const isAbbr = isAbbreviation(lastWord) || isNumDot;
+      const startsWithLowercase = /^[a-zа-я]/.test(seg.trim());
+
+      if (isAbbr || startsWithLowercase) {
+        current += seg;
+      } else {
+        sentences.push(current.trim());
+        current = seg;
+      }
+    }
+  }
+
+  if (current && current.trim().length > 0) {
+    sentences.push(current.trim());
+  }
+
+  return sentences;
+}
+
+function fallbackSentenceSplit(text) {
+  return text.split(/(?<=[.!?])\s+/).filter((s) => s.trim().length > 0);
+}

package/mcp-server/memory.js

@@ -3,12 +3,36 @@ import { existsSync } from "fs";
 import { join, basename } from "path";
 import { homedir } from "os";
 
-const CONFIG_DIR = process.env.MEMORY_DIR || process.env.OPENCODE_CONFIG_DIR || join(homedir(), ".config", "opencode");
-export const MEMORY_DIR = join(CONFIG_DIR, "memory");
+function resolveMemoryDir() {
+  if (process.env.MEMORY_DIR) return process.env.MEMORY_DIR;
+  if (process.env.OPENCODE_CONFIG_DIR) return join(process.env.OPENCODE_CONFIG_DIR, "memory");
+
+  const legacyDir = join(homedir(), ".config", "opencode", "memory");
+  if (existsSync(legacyDir)) {
+    return legacyDir;
+  }
+
+  if (process.platform === "win32") {
+    const appData = process.env.LOCALAPPDATA || process.env.APPDATA || join(homedir(), "AppData", "Local");
+    return join(appData, "opencode", "memory");
+  }
+
+  const configHome = process.env.XDG_CONFIG_HOME || join(homedir(), ".config");
+  return join(configHome, "opencode", "memory");
+}
+
+export const MEMORY_DIR = resolveMemoryDir();
 export const GLOBAL_KEY = "global";
 
 export async function ensureDir() {
   if (!existsSync(MEMORY_DIR)) await mkdir(MEMORY_DIR, { recursive: true });
+  const storageDir = join(MEMORY_DIR, "storage");
+  const blobsDir = join(storageDir, "blobs");
+  const modelsDir = join(storageDir, "models");
+  const exportsDir = join(MEMORY_DIR, "exports");
+  if (!existsSync(blobsDir)) await mkdir(blobsDir, { recursive: true });
+  if (!existsSync(modelsDir)) await mkdir(modelsDir, { recursive: true });
+  if (!existsSync(exportsDir)) await mkdir(exportsDir, { recursive: true });
 }
 
 export function projectName(worktree, directory) {

package/mcp-server/ml/model_manager.js

@@ -0,0 +1,185 @@
+import { MODELS_DIR } from "../db/database.js";
+
+let extractorInstance = null;
+let loadedModelName = null;
+
+let rerankerInstance = null;
+let loadedRerankerName = null;
+
+import { getConfig } from "../config/config_manager.js";
+
+export async function getExtractor(modelName = null, progressCallback = null) {
+  const targetModel = modelName || getConfig().embeddingModel || "Xenova/multilingual-e5-small";
+
+  if (extractorInstance && loadedModelName === targetModel) {
+    return extractorInstance;
+  }
+
+  const { pipeline, env } = await import("@xenova/transformers");
+
+  // Fix ONNX Runtime Node.js WASM execution provider warning
+  try {
+    const { executionProviders } = await import("@xenova/transformers/src/backends/onnx.js");
+    const wasmIdx = executionProviders.indexOf("wasm");
+    if (wasmIdx !== -1) {
+      executionProviders.splice(wasmIdx, 1);
+    }
+  } catch {}
+
+  env.cacheDir = MODELS_DIR;
+  env.allowLocalModels = true;
+  env.allowRemoteModels = true;
+  env.remoteHost = "https://huggingface.co";
+  env.remotePathTemplate = "{model}/resolve/{revision}/";
+
+  const pipelineOpts = { quantized: true };
+  if (progressCallback) {
+    pipelineOpts.progress_callback = progressCallback;
+  }
+
+  try {
+    extractorInstance = await pipeline("feature-extraction", targetModel, pipelineOpts);
+    loadedModelName = targetModel;
+  } catch (err) {
+    console.warn(`HuggingFace model load failed for ${targetModel}: ${err.message}. Falling back to default Xenova/multilingual-e5-small...`);
+    if (targetModel !== "Xenova/multilingual-e5-small") {
+      extractorInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", pipelineOpts);
+      loadedModelName = "Xenova/multilingual-e5-small";
+    } else {
+      throw err;
+    }
+  }
+
+  return extractorInstance;
+}
+
+export function formatInputText(text, isQuery = false, modelName = null, instruction = null) {
+  if (!text) return "";
+  const targetModel = modelName || getConfig().embeddingModel || "Xenova/multilingual-e5-small";
+  const name = targetModel.toLowerCase();
+
+  let cleanText = text.trim();
+  if (cleanText.startsWith("passage: ")) {
+    cleanText = cleanText.substring(9).trim();
+  } else if (cleanText.startsWith("query: ")) {
+    cleanText = cleanText.substring(7).trim();
+  }
+
+  // 1. E5 Model Family (multilingual-e5-small, multilingual-e5-large, etc.)
+  if (name.includes("e5")) {
+    const isInstructModel = name.includes("-instruct");
+    if (isQuery) {
+      if (instruction && instruction.trim() && isInstructModel) {
+        return `Instruct: ${instruction.trim()}\nQuery: ${cleanText}`;
+      }
+      return `query: ${cleanText}`;
+    }
+    return `passage: ${cleanText}`;
+  }
+
+  // 2. BGE Model Family (bge-m3, bge-small-en-v1.5, etc.)
+  if (name.includes("bge")) {
+    if (isQuery) {
+      if (instruction && instruction.trim()) {
+        return `Represent this sentence for searching relevant passages: ${instruction.trim()} ${cleanText}`;
+      }
+      return `Represent this sentence for searching relevant passages: ${cleanText}`;
+    }
+    return cleanText;
+  }
+
+  // 3. MiniLM / Standard models (no prefixes)
+  return cleanText;
+}
+
+export async function embedText(text, isQuery = false, modelName = null, progressCallback = null, instruction = null) {
+  const targetModel = modelName || getConfig().embeddingModel || "Xenova/multilingual-e5-small";
+  const extractor = await getExtractor(targetModel, progressCallback);
+  const formattedText = formatInputText(text, isQuery, targetModel, instruction);
+
+  const output = await extractor(formattedText, {
+    pooling: "mean",
+    normalize: true,
+  });
+
+  return new Float32Array(output.data);
+}
+
+export async function getReranker(modelName = "Xenova/bge-reranker-base", progressCallback = null) {
+  if (rerankerInstance && loadedRerankerName === modelName) {
+    return rerankerInstance;
+  }
+
+  const { pipeline, env } = await import("@xenova/transformers");
+  env.cacheDir = MODELS_DIR;
+
+  const pipelineOpts = { quantized: true };
+  if (progressCallback) {
+    pipelineOpts.progress_callback = progressCallback;
+  }
+
+  try {
+    rerankerInstance = await pipeline("text-classification", modelName, pipelineOpts);
+    loadedRerankerName = modelName;
+  } catch (err) {
+    console.warn(`Failed to load reranker model ${modelName}: ${err.message}`);
+    return null;
+  }
+  return rerankerInstance;
+}
+
+export async function preloadModel(modelName, type = "embedding", progressCallback = null) {
+  if (type === "reranker") {
+    return await getReranker(modelName, progressCallback);
+  }
+  return await getExtractor(modelName, progressCallback);
+}
+
+export async function rerankHits(query, hits, rerankerModelName = "Xenova/bge-reranker-base") {
+  if (!hits || hits.length === 0) return hits;
+  const classifier = await getReranker(rerankerModelName);
+  if (!classifier) return hits;
+
+  const reranked = [];
+  for (const hit of hits) {
+    try {
+      const input = `${query} | ${hit.content}`;
+      const res = await classifier(input);
+      const score = res && res[0] ? res[0].score : hit.rsf_score || hit.rrf_score || 0;
+      reranked.push({
+        ...hit,
+        rerank_score: score,
+      });
+    } catch (err) {
+      reranked.push(hit);
+    }
+  }
+
+  reranked.sort((a, b) => (b.rerank_score || 0) - (a.rerank_score || 0));
+  return reranked;
+}
+
+export function vectorToBuffer(float32Array) {
+  return Buffer.from(float32Array.buffer, float32Array.byteOffset, float32Array.byteLength);
+}
+
+export function bufferToVector(buffer) {
+  return new Float32Array(
+    buffer.buffer,
+    buffer.byteOffset,
+    buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
+  );
+}
+
+export function cosineSimilarity(vecA, vecB) {
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < vecA.length; i++) {
+    dotProduct += vecA[i] * vecB[i];
+    normA += vecA[i] * vecA[i];
+    normB += vecB[i] * vecB[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/mcp-server/retrieval/retriever.js

@@ -0,0 +1,299 @@
+import { getDatabase } from "../db/database.js";
+import { embedText, bufferToVector, cosineSimilarity, rerankHits } from "../ml/model_manager.js";
+import { getRelatedSymbols } from "../graph/graph_extractor.js";
+import { getConfig } from "../config/config_manager.js";
+
+export function sanitizeFtsQuery(query) {
+  if (!query) return "";
+  const cleaned = query.replace(/[^a-zA-Z0-9_\u0400-\u04FF\s]/g, " ").trim();
+  const words = cleaned.split(/\s+/).filter((w) => w.length > 0);
+  if (words.length === 0) return "";
+  return words.join(" OR ");
+}
+
+export function bm25Search(db, query, limit = 30) {
+  const ftsQuery = sanitizeFtsQuery(query);
+  if (!ftsQuery) return [];
+
+  try {
+    const stmt = db.prepare(`
+      SELECT id, content, breadcrumbs, rank
+      FROM micro_chunks_fts
+      WHERE micro_chunks_fts MATCH ?
+      ORDER BY rank
+      LIMIT ?;
+    `);
+    const rows = stmt.all(ftsQuery, limit);
+    return rows.map((r, i) => ({
+      id: r.id,
+      content: r.content,
+      breadcrumbs: r.breadcrumbs,
+      bm25_rank: i + 1,
+      fts_rank: r.rank,
+    }));
+  } catch (err) {
+    console.warn("FTS5 query execution failed:", err.message);
+    return [];
+  }
+}
+
+export function vectorSearch(db, queryVector, limit = 30, minSim = 0.25) {
+  if (!queryVector || queryVector.length === 0) return [];
+
+  const stmt = db.prepare(`
+    SELECT m.id, m.section_id, m.doc_id, m.content, m.vector, s.breadcrumbs
+    FROM micro_chunks m
+    JOIN sections s ON m.section_id = s.id;
+  `);
+  const rows = stmt.all();
+
+  const scored = [];
+  for (const r of rows) {
+    const vec = bufferToVector(r.vector);
+    const sim = cosineSimilarity(queryVector, vec);
+    if (!isNaN(sim) && sim >= minSim) {
+      scored.push({
+        id: r.id,
+        section_id: r.section_id,
+        doc_id: r.doc_id,
+        content: r.content,
+        breadcrumbs: r.breadcrumbs,
+        cosine_sim: sim,
+      });
+    }
+  }
+
+  scored.sort((a, b) => b.cosine_sim - a.cosine_sim);
+  const top = scored.slice(0, limit);
+
+  return top.map((item, idx) => ({
+    ...item,
+    vector_rank: idx + 1,
+  }));
+}
+
+export function rrfFusion(bm25Hits, vectorHits, k = 60, scoreThreshold = 0.01) {
+  const scoreMap = new Map();
+
+  bm25Hits.forEach((hit) => {
+    const existing = scoreMap.get(hit.id) || {
+      id: hit.id,
+      content: hit.content,
+      breadcrumbs: hit.breadcrumbs,
+      rrf_score: 0,
+      bm25_rank: null,
+      vector_rank: null,
+    };
+    existing.bm25_rank = hit.bm25_rank;
+    existing.rrf_score += 1.0 / (k + hit.bm25_rank);
+    scoreMap.set(hit.id, existing);
+  });
+
+  vectorHits.forEach((hit) => {
+    const existing = scoreMap.get(hit.id) || {
+      id: hit.id,
+      content: hit.content,
+      breadcrumbs: hit.breadcrumbs,
+      rrf_score: 0,
+      bm25_rank: null,
+      vector_rank: null,
+    };
+    existing.vector_rank = hit.vector_rank;
+    existing.cosine_sim = hit.cosine_sim;
+    existing.rrf_score += 1.0 / (k + hit.vector_rank);
+    scoreMap.set(hit.id, existing);
+  });
+
+  const merged = Array.from(scoreMap.values()).map((item) => ({
+    ...item,
+    score: item.rrf_score,
+  }));
+  merged.sort((a, b) => b.rrf_score - a.rrf_score);
+
+  return merged.filter((item) => item.rrf_score >= scoreThreshold);
+}
+
+export function rsfFusion(bm25Hits, vectorHits, alpha = 0.5, scoreThreshold = 0.01) {
+  const scoreMap = new Map();
+
+  let minFts = Infinity;
+  let maxFts = -Infinity;
+  bm25Hits.forEach((hit) => {
+    const r = hit.fts_rank !== undefined ? hit.fts_rank : -hit.bm25_rank;
+    if (r < minFts) minFts = r;
+    if (r > maxFts) maxFts = r;
+  });
+
+  let minSim = Infinity;
+  let maxSim = -Infinity;
+  vectorHits.forEach((hit) => {
+    const sim = hit.cosine_sim || 0;
+    if (sim < minSim) minSim = sim;
+    if (sim > maxSim) maxSim = sim;
+  });
+
+  bm25Hits.forEach((hit) => {
+    const r = hit.fts_rank !== undefined ? hit.fts_rank : -hit.bm25_rank;
+    let normLexical = 1.0;
+    if (maxFts > minFts) {
+      normLexical = (maxFts - r) / (maxFts - minFts);
+    }
+    scoreMap.set(hit.id, {
+      id: hit.id,
+      content: hit.content,
+      breadcrumbs: hit.breadcrumbs,
+      bm25_rank: hit.bm25_rank,
+      vector_rank: null,
+      cosine_sim: null,
+      norm_lexical: normLexical,
+      norm_semantic: 0.0,
+    });
+  });
+
+  vectorHits.forEach((hit) => {
+    const existing = scoreMap.get(hit.id) || {
+      id: hit.id,
+      content: hit.content,
+      breadcrumbs: hit.breadcrumbs,
+      bm25_rank: null,
+      vector_rank: null,
+      cosine_sim: hit.cosine_sim,
+      norm_lexical: 0.0,
+      norm_semantic: 0.0,
+    };
+    existing.vector_rank = hit.vector_rank;
+    existing.cosine_sim = hit.cosine_sim;
+
+    let normSemantic = hit.cosine_sim || 0;
+    if (maxSim > minSim) {
+      normSemantic = (hit.cosine_sim - minSim) / (maxSim - minSim);
+    }
+    existing.norm_semantic = normSemantic;
+
+    scoreMap.set(hit.id, existing);
+  });
+
+  const merged = Array.from(scoreMap.values()).map((item) => {
+    const rsfScore = alpha * item.norm_semantic + (1.0 - alpha) * item.norm_lexical;
+    return {
+      ...item,
+      rsf_score: rsfScore,
+      score: rsfScore,
+    };
+  });
+
+  merged.sort((a, b) => b.rsf_score - a.rsf_score);
+  return merged.filter((item) => item.rsf_score >= scoreThreshold);
+}
+
+export async function hybridQuery({
+  query,
+  limit = 5,
+  scoreThreshold = 0.01,
+  customDb = null,
+  includeGraphContext = true,
+  fusionAlgorithm = null,
+  alpha = null,
+  embeddingModel = null,
+  rerankerModel = null,
+  rerankerEnabled = null,
+  instruction = null,
+}) {
+  const db = customDb || getDatabase();
+  const activeConfig = getConfig();
+
+  const algo = fusionAlgorithm || activeConfig.fusionAlgorithm || "rsf";
+  const alphaWeight = alpha !== null && alpha !== undefined ? alpha : (activeConfig.alpha ?? 0.5);
+  const embModel = embeddingModel || activeConfig.embeddingModel || "Xenova/multilingual-e5-small";
+  const useReranker = rerankerEnabled !== null ? rerankerEnabled : (activeConfig.rerankerEnabled ?? false);
+  const rerankModelName = rerankerModel || activeConfig.rerankerModel || "Xenova/bge-reranker-base";
+
+  let fusedHits = [];
+
+  if (algo === "lexical_only" || algo === "bm25_only") {
+    const bm25Hits = bm25Search(db, query, limit * 4);
+    fusedHits = bm25Hits.map((hit) => ({
+      ...hit,
+      score: 1.0 / hit.bm25_rank,
+    }));
+  } else if (algo === "semantic_only" || algo === "vector_only") {
+    const queryVector = await embedText(query, true, embModel, null, instruction);
+    const vectorHits = vectorSearch(db, queryVector, limit * 4, 0.10);
+    fusedHits = vectorHits.map((hit) => ({
+      ...hit,
+      score: hit.cosine_sim,
+    }));
+  } else if (algo === "rrf") {
+    const bm25Hits = bm25Search(db, query, 30);
+    const queryVector = await embedText(query, true, embModel, null, instruction);
+    const vectorHits = vectorSearch(db, queryVector, 30, 0.10);
+    fusedHits = rrfFusion(bm25Hits, vectorHits, 60, scoreThreshold);
+  } else {
+    // Default: RSF
+    const bm25Hits = bm25Search(db, query, 30);
+    const queryVector = await embedText(query, true, embModel, null, instruction);
+    const vectorHits = vectorSearch(db, queryVector, 30, 0.10);
+    fusedHits = rsfFusion(bm25Hits, vectorHits, alphaWeight, scoreThreshold);
+  }
+
+  if (useReranker && rerankModelName !== "none") {
+    fusedHits = await rerankHits(query, fusedHits, rerankModelName);
+  }
+
+  // Parent-Child Rollup: Deduplicate hits sharing the same medium_id or section_id to prevent noise
+  const parentDeduplicatedHits = [];
+  const seenParents = new Set();
+  const parentLookupStmt = db.prepare(`
+    SELECT medium_id, section_id FROM micro_chunks WHERE id = ?;
+  `);
+
+  for (const hit of fusedHits) {
+    const row = parentLookupStmt.get(hit.id);
+    const parentKey = row ? (row.medium_id || row.section_id) : hit.id;
+    if (!seenParents.has(parentKey)) {
+      seenParents.add(parentKey);
+      parentDeduplicatedHits.push(hit);
+    }
+  }
+
+  const topHits = parentDeduplicatedHits.slice(0, limit);
+  const results = [];
+
+  const secStmt = db.prepare(`
+    SELECT s.id as section_id, s.heading, s.breadcrumbs, s.content as section_content,
+           med.content as medium_content, d.title as doc_title, d.path as doc_path
+    FROM micro_chunks m
+    JOIN sections s ON m.section_id = s.id
+    JOIN documents d ON m.doc_id = d.id
+    LEFT JOIN medium_chunks med ON m.medium_id = med.id
+    WHERE m.id = ?;
+  `);
+
+  for (const hit of topHits) {
+    const detail = secStmt.get(hit.id);
+    if (!detail) continue;
+
+    let symbols = [];
+    if (includeGraphContext) {
+      symbols = getRelatedSymbols(db, detail.section_id);
+    }
+
+    results.push({
+      chunk_id: hit.id,
+      doc_title: detail.doc_title,
+      doc_path: detail.doc_path,
+      heading: detail.heading,
+      breadcrumbs: detail.breadcrumbs,
+      snippet: hit.content,
+      paragraph_context: detail.medium_content || hit.content,
+      full_section_content: detail.section_content,
+      score: parseFloat((hit.score || 0).toFixed(4)),
+      rsf_score: hit.rsf_score ? parseFloat(hit.rsf_score.toFixed(4)) : null,
+      rrf_score: hit.rrf_score ? parseFloat(hit.rrf_score.toFixed(4)) : null,
+      cosine_sim: hit.cosine_sim ? parseFloat(hit.cosine_sim.toFixed(4)) : null,
+      defined_symbols: symbols,
+    });
+  }
+
+  return results;
+}

package/mcp-server/setup.js

@@ -32,13 +32,19 @@ export async function runSetup() {
         } catch (e) {}
       }
       if (!Array.isArray(config.plugin)) config.plugin = [];
+      // Clean up legacy / incorrect plugin entry names
+      const obsoleteNames = ["opencode-memory-plugin", "memory_plugin", "memory-plugin"];
+      config.plugin = config.plugin.filter((p) => !obsoleteNames.includes(p));
       if (!config.plugin.includes("@lotargo/memory_plugin")) {
         config.plugin.push("@lotargo/memory_plugin");
-        await writeFile(opencodeConfigPath, JSON.stringify(config, null, 2));
-        console.log("  [OK] OpenCode: added plugin to ~/.config/opencode/opencode.json");
-      } else {
-        console.log("  [INFO] OpenCode: already configured");
       }
+      // Clean up legacy mcp-helper.js standalone file plugin if present
+      const legacyPluginFile = join(opencodeDir, "plugins", "mcp-helper.js");
+      if (existsSync(legacyPluginFile)) {
+        try { const { unlink } = await import("fs/promises"); await unlink(legacyPluginFile); } catch (e) {}
+      }
+      await writeFile(opencodeConfigPath, JSON.stringify(config, null, 2));
+      console.log("  [OK] OpenCode: configured plugin in ~/.config/opencode/opencode.json");
       configuredCount++;
     } catch (err) {
       console.log("  [SKIP] OpenCode setup skipped:", err.message);

package/mcp-server/storage/blob_store.js

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import { gzipSync, gunzipSync } from "node:zlib";
+import { readFile, writeFile, mkdir, unlink, access } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { BLOBS_DIR } from "../db/database.js";
+
+export function hashContent(data) {
+  const hash = createHash("sha256");
+  hash.update(data);
+  return hash.digest("hex");
+}
+
+export function getBlobPath(hash, baseDir = BLOBS_DIR) {
+  const prefix = hash.substring(0, 2);
+  return join(baseDir, prefix, `${hash}.raw.gz`);
+}
+
+export async function blobExists(hash, baseDir = BLOBS_DIR) {
+  const blobPath = getBlobPath(hash, baseDir);
+  return existsSync(blobPath);
+}
+
+export async function saveBlob(content, baseDir = BLOBS_DIR) {
+  const buffer = typeof content === "string" ? Buffer.from(content, "utf-8") : content;
+  const hash = hashContent(buffer);
+  const blobPath = getBlobPath(hash, baseDir);
+
+  if (existsSync(blobPath)) {
+    return { hash, size: buffer.length, path: blobPath, deduplicated: true };
+  }
+
+  const parentDir = join(blobPath, "..");
+  if (!existsSync(parentDir)) {
+    await mkdir(parentDir, { recursive: true });
+  }
+
+  const compressed = gzipSync(buffer);
+  await writeFile(blobPath, compressed);
+
+  return { hash, size: buffer.length, path: blobPath, deduplicated: false };
+}
+
+export async function readBlob(hash, baseDir = BLOBS_DIR) {
+  const blobPath = getBlobPath(hash, baseDir);
+  if (!existsSync(blobPath)) {
+    throw new Error(`Blob not found for hash: ${hash}`);
+  }
+
+  const compressed = await readFile(blobPath);
+  const decompressed = gunzipSync(compressed);
+  return decompressed.toString("utf-8");
+}
+
+export async function deleteBlob(hash, baseDir = BLOBS_DIR) {
+  const blobPath = getBlobPath(hash, baseDir);
+  if (existsSync(blobPath)) {
+    await unlink(blobPath);
+    return true;
+  }
+  return false;
+}