cognitive-core 0.1.0 → 0.2.0

package/references/minimem/src/internal.ts

@@ -0,0 +1,299 @@
+import crypto from "node:crypto";
+import fsSync from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Debug function type for optional logging
+ */
+export type DebugFn = (message: string, data?: Record<string, unknown>) => void;
+
+/**
+ * Log an error with context (for debugging).
+ * Only logs if a debug function is provided.
+ *
+ * @param context - A short identifier for where the error occurred
+ * @param error - The error object or message
+ * @param debug - Optional debug function to log to
+ */
+export function logError(
+  context: string,
+  error: unknown,
+  debug?: DebugFn
+): void {
+  if (!debug) return;
+
+  const message = error instanceof Error ? error.message : String(error);
+  debug(`[${context}] Error: ${message}`);
+}
+
+export type MemoryFileEntry = {
+  path: string;
+  absPath: string;
+  mtimeMs: number;
+  size: number;
+  hash: string;
+};
+
+export type MemoryChunk = {
+  startLine: number;
+  endLine: number;
+  text: string;
+  hash: string;
+};
+
+/**
+ * Ensure a directory exists, creating it if necessary.
+ *
+ * @param dir - The directory path to ensure exists
+ * @param debug - Optional debug function for logging errors
+ * @returns The directory path
+ */
+export function ensureDir(dir: string, debug?: DebugFn): string {
+  try {
+    fsSync.mkdirSync(dir, { recursive: true });
+  } catch (error) {
+    // Only swallow EEXIST errors (directory already exists)
+    // Log other errors for debugging
+    const nodeError = error as NodeJS.ErrnoException;
+    if (nodeError.code !== "EEXIST") {
+      logError("ensureDir", error, debug);
+    }
+  }
+  return dir;
+}
+
+export function normalizeRelPath(value: string): string {
+  const trimmed = value.trim().replace(/^[./]+/, "");
+  return trimmed.replace(/\\/g, "/");
+}
+
+export function isMemoryPath(relPath: string): boolean {
+  const normalized = normalizeRelPath(relPath);
+  if (!normalized) return false;
+  if (normalized === "MEMORY.md" || normalized === "memory.md") return true;
+  return normalized.startsWith("memory/");
+}
+
+async function exists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function walkDir(dir: string, files: string[]) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkDir(full, files);
+      continue;
+    }
+    if (!entry.isFile()) continue;
+    if (!entry.name.endsWith(".md")) continue;
+    files.push(full);
+  }
+}
+
+export async function listMemoryFiles(memoryDir: string): Promise<string[]> {
+  const result: string[] = [];
+  const memoryFile = path.join(memoryDir, "MEMORY.md");
+  const altMemoryFile = path.join(memoryDir, "memory.md");
+  const hasUpper = await exists(memoryFile);
+  const hasLower = await exists(altMemoryFile);
+
+  // Prevent ambiguity: both MEMORY.md and memory.md cannot coexist
+  // (unless they resolve to the same file on case-insensitive filesystems)
+  if (hasUpper && hasLower) {
+    let upperReal = memoryFile;
+    let lowerReal = altMemoryFile;
+    try { upperReal = await fs.realpath(memoryFile); } catch {}
+    try { lowerReal = await fs.realpath(altMemoryFile); } catch {}
+    if (upperReal !== lowerReal) {
+      throw new Error(
+        `Both MEMORY.md and memory.md exist in ${memoryDir}. ` +
+        `Please remove one to avoid ambiguity.`
+      );
+    }
+    // Same file (case-insensitive FS) — only include once
+    result.push(memoryFile);
+  } else if (hasUpper) {
+    result.push(memoryFile);
+  } else if (hasLower) {
+    result.push(altMemoryFile);
+  }
+
+  const memorySubDir = path.join(memoryDir, "memory");
+  if (await exists(memorySubDir)) {
+    await walkDir(memorySubDir, result);
+  }
+  if (result.length <= 1) return result;
+  const seen = new Set<string>();
+  const deduped: string[] = [];
+  for (const entry of result) {
+    let key = entry;
+    try {
+      key = await fs.realpath(entry);
+    } catch {}
+    if (seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(entry);
+  }
+  return deduped;
+}
+
+export function hashText(value: string): string {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+export async function buildFileEntry(
+  absPath: string,
+  memoryDir: string,
+): Promise<MemoryFileEntry> {
+  const stat = await fs.stat(absPath);
+  const content = await fs.readFile(absPath, "utf-8");
+  const hash = hashText(content);
+  return {
+    path: path.relative(memoryDir, absPath).replace(/\\/g, "/"),
+    absPath,
+    mtimeMs: stat.mtimeMs,
+    size: stat.size,
+    hash,
+  };
+}
+
+/**
+ * Strip content between <private>...</private> tags.
+ * Replaces private blocks with the same number of empty lines to preserve line numbering.
+ */
+export function stripPrivateContent(content: string): string {
+  return content.replace(/<private>[\s\S]*?<\/private>/gi, (match) => {
+    const lineCount = match.split("\n").length;
+    return "\n".repeat(lineCount - 1);
+  });
+}
+
+export function chunkMarkdown(
+  content: string,
+  chunking: { tokens: number; overlap: number },
+): MemoryChunk[] {
+  const stripped = stripPrivateContent(content);
+  const lines = stripped.split("\n");
+  if (lines.length === 0) return [];
+  const maxChars = Math.max(32, chunking.tokens * 4);
+  const overlapChars = Math.max(0, chunking.overlap * 4);
+  const chunks: MemoryChunk[] = [];
+
+  let current: Array<{ line: string; lineNo: number }> = [];
+  let currentChars = 0;
+
+  const flush = () => {
+    if (current.length === 0) return;
+    const firstEntry = current[0];
+    const lastEntry = current[current.length - 1];
+    if (!firstEntry || !lastEntry) return;
+    const text = current.map((entry) => entry.line).join("\n");
+    const startLine = firstEntry.lineNo;
+    const endLine = lastEntry.lineNo;
+    chunks.push({
+      startLine,
+      endLine,
+      text,
+      hash: hashText(text),
+    });
+  };
+
+  const carryOverlap = () => {
+    if (overlapChars <= 0 || current.length === 0) {
+      current = [];
+      currentChars = 0;
+      return;
+    }
+    let acc = 0;
+    const kept: Array<{ line: string; lineNo: number }> = [];
+    for (let i = current.length - 1; i >= 0; i -= 1) {
+      const entry = current[i];
+      if (!entry) continue;
+      acc += entry.line.length + 1;
+      kept.unshift(entry);
+      if (acc >= overlapChars) break;
+    }
+    current = kept;
+    currentChars = kept.reduce((sum, entry) => sum + entry.line.length + 1, 0);
+  };
+
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i] ?? "";
+    const lineNo = i + 1;
+    const segments: string[] = [];
+    if (line.length === 0) {
+      segments.push("");
+    } else {
+      for (let start = 0; start < line.length; start += maxChars) {
+        segments.push(line.slice(start, start + maxChars));
+      }
+    }
+    for (const segment of segments) {
+      const lineSize = segment.length + 1;
+      if (currentChars + lineSize > maxChars && current.length > 0) {
+        flush();
+        carryOverlap();
+      }
+      current.push({ line: segment, lineNo });
+      currentChars += lineSize;
+    }
+  }
+  flush();
+  return chunks;
+}
+
+/**
+ * Extract metadata from a chunk's text content.
+ * Looks for HTML comments like <!-- type: decision --> in the chunk.
+ */
+export function extractChunkMetadata(text: string): { type?: string } {
+  const typeMatch = text.match(/<!--\s*type:\s*([\w-]+)\s*-->/i);
+  return typeMatch ? { type: typeMatch[1].toLowerCase() } : {};
+}
+
+export function parseEmbedding(raw: string): number[] {
+  try {
+    const parsed = JSON.parse(raw) as number[];
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length === 0 || b.length === 0) return 0;
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < len; i += 1) {
+    const av = a[i] ?? 0;
+    const bv = b[i] ?? 0;
+    dot += av * bv;
+    normA += av * av;
+    normB += bv * bv;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+export function truncateUtf16Safe(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  return text.slice(0, maxChars);
+}
+
+/**
+ * Convert a numeric embedding vector to a Buffer suitable for sqlite-vec storage.
+ * Uses Float32 encoding, which is the format expected by sqlite-vec's vector functions.
+ */
+export function vectorToBlob(embedding: number[]): Buffer {
+  return Buffer.from(new Float32Array(embedding).buffer);
+}