cognitive-core 0.1.0 → 0.2.0

package/references/minimem/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "minimem",
+  "version": "0.0.3",
+  "description": "A lightweight file-based memory system with vector search for AI agents",
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup && npm run postbuild",
+    "postbuild": "node scripts/postbuild.js",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:integration": "npx tsx --test src/__tests__/minimem.integration.test.ts",
+    "test:cli": "npx tsx --test src/cli/__tests__/commands.test.ts",
+    "test:all": "npm run test && npm run test:integration && npm run test:cli",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "memory",
+    "vector-search",
+    "embeddings",
+    "sqlite",
+    "ai-agent",
+    "rag",
+    "semantic-search"
+  ],
+  "author": "Alex Ngai",
+  "license": "MIT",
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "bin": {
+    "minimem": "./dist/cli/index.js"
+  },
+  "dependencies": {
+    "chokidar": "^4.0.3",
+    "commander": "^12.0.0",
+    "minimatch": "^10.1.1",
+    "sqlite-vec": "^0.1.6"
+  },
+  "optionalDependencies": {
+    "node-llama-cpp": "^3.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^2.0.0"
+  }
+}

package/references/minimem/scripts/postbuild.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+/**
+ * Cross-platform postbuild script
+ *
+ * Fixes the sqlite import in the CLI bundle.
+ * esbuild/tsup strips the "node:" prefix from built-in modules,
+ * but node:sqlite requires it.
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const cliPath = join(__dirname, "../dist/cli/index.js");
+
+try {
+  let content = readFileSync(cliPath, "utf-8");
+
+  // Fix sqlite import (esbuild strips the node: prefix)
+  const before = content;
+  content = content.replace(/from "sqlite"/g, 'from "node:sqlite"');
+  content = content.replace(/from 'sqlite'/g, "from 'node:sqlite'");
+
+  if (content !== before) {
+    writeFileSync(cliPath, content);
+    console.log("✓ Fixed node:sqlite import in dist/cli/index.js");
+  } else {
+    console.log("✓ No sqlite import fixes needed");
+  }
+} catch (error) {
+  console.error("Error running postbuild:", error.message);
+  process.exit(1);
+}

package/references/minimem/src/__tests__/edge-cases.test.ts

@@ -0,0 +1,371 @@
+/**
+ * Edge case tests
+ *
+ * Tests for boundary conditions and unusual inputs:
+ * - Empty files
+ * - Unicode content (emoji, CJK, RTL)
+ * - Very large files
+ * - Files with only frontmatter
+ * - Windows line endings
+ * - Path validation edge cases
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  chunkMarkdown,
+  hashText,
+  buildFileEntry,
+  listMemoryFiles,
+  isMemoryPath,
+  cosineSimilarity,
+  truncateUtf16Safe,
+} from "../internal.js";
+import {
+  parseFrontmatter,
+  serializeFrontmatter,
+  addSessionToContent,
+  type SessionContext,
+} from "../session.js";
+
+describe("Edge case: empty files", () => {
+  let tempDir: string;
+
+  beforeEach(async () => {
+    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "minimem-edge-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempDir, { recursive: true, force: true });
+  });
+
+  it("listMemoryFiles includes empty .md files", async () => {
+    await fs.writeFile(path.join(tempDir, "MEMORY.md"), "");
+    const files = await listMemoryFiles(tempDir);
+    expect(files).toHaveLength(1);
+  });
+
+  it("chunkMarkdown handles empty string", () => {
+    const chunks = chunkMarkdown("", { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeLessThanOrEqual(1);
+  });
+
+  it("chunkMarkdown handles single newline", () => {
+    const chunks = chunkMarkdown("\n", { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeLessThanOrEqual(1);
+  });
+
+  it("chunkMarkdown handles whitespace-only content", () => {
+    const chunks = chunkMarkdown("   \n   \n   ", { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+    // Each chunk should have text defined
+    for (const chunk of chunks) {
+      expect(chunk.text).toBeDefined();
+      expect(chunk.hash).toBeDefined();
+    }
+  });
+
+  it("buildFileEntry handles empty file", async () => {
+    const filePath = path.join(tempDir, "empty.md");
+    await fs.writeFile(filePath, "");
+
+    const entry = await buildFileEntry(filePath, tempDir);
+    expect(entry.path).toBe("empty.md");
+    expect(entry.size).toBe(0);
+    expect(entry.hash).toBe(hashText(""));
+  });
+});
+
+describe("Edge case: unicode content", () => {
+  it("chunkMarkdown handles emoji content", () => {
+    const content = "# 🚀 Project Launch\n\n📝 Notes about the launch 🎉\n\n## ✅ Tasks\n- 🔥 Deploy\n- 🧪 Test";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(chunks[0].text).toContain("🚀");
+  });
+
+  it("chunkMarkdown handles CJK characters", () => {
+    const content = "# 项目笔记\n\n这是一个测试文件。\n\n## 会议记录\n今天讨论了新功能。";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(chunks[0].text).toContain("项目笔记");
+  });
+
+  it("chunkMarkdown handles RTL text (Arabic)", () => {
+    const content = "# ملاحظات المشروع\n\nهذا ملف اختبار.\n\n## سجل الاجتماعات";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(chunks[0].text).toContain("ملاحظات");
+  });
+
+  it("chunkMarkdown handles mixed scripts", () => {
+    const content = "# Mixed: English, 中文, العربية, Ελληνικά\n\nContent with мнoго different scripts.";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+  });
+
+  it("hashText produces consistent hashes for unicode", () => {
+    const emoji = "🚀🎉";
+    const h1 = hashText(emoji);
+    const h2 = hashText(emoji);
+    expect(h1).toBe(h2);
+    expect(h1.length).toBe(64);
+  });
+
+  it("parseFrontmatter handles unicode values", () => {
+    const content = '---\nsession:\n  source: "テスト"\ncreated: 2024-01-01\n---\n# 内容';
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter?.session?.source).toBe("テスト");
+    expect(body).toBe("# 内容");
+  });
+
+  it("truncateUtf16Safe truncates at char boundary", () => {
+    const emoji = "🚀🎉🔥💡"; // Each emoji is 2 UTF-16 code units
+    const truncated = truncateUtf16Safe(emoji, 2);
+    // Should truncate at character count
+    expect(truncated.length).toBe(2);
+  });
+});
+
+describe("Edge case: large files", () => {
+  it("chunkMarkdown splits large content into multiple chunks", () => {
+    // Generate ~100KB of content
+    const lines: string[] = [];
+    for (let i = 0; i < 2000; i++) {
+      lines.push(`Line ${i}: This is a test line with enough content to fill chunks properly.`);
+    }
+    const content = lines.join("\n");
+
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(10);
+
+    // Verify line numbers are monotonically increasing
+    for (let i = 1; i < chunks.length; i++) {
+      expect(chunks[i].startLine).toBeGreaterThanOrEqual(chunks[i - 1].startLine);
+    }
+
+    // Verify all content is captured (first and last lines)
+    expect(chunks[0].text).toContain("Line 0");
+    expect(chunks[chunks.length - 1].text).toContain("Line 1999");
+  });
+
+  it("chunkMarkdown handles single very long line", () => {
+    const longLine = "a".repeat(10000);
+    const chunks = chunkMarkdown(longLine, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(1);
+
+    // All chunks should have valid hash and line numbers
+    for (const chunk of chunks) {
+      expect(chunk.hash).toBeDefined();
+      expect(chunk.hash.length).toBe(64);
+      expect(chunk.startLine).toBe(1); // Single line = always line 1
+      expect(chunk.endLine).toBe(1);
+    }
+
+    // Total text across chunks (minus overlap) should cover original content
+    // Just verify first and last segments are present
+    expect(chunks[0].text.length).toBeGreaterThan(0);
+    expect(chunks[chunks.length - 1].text.length).toBeGreaterThan(0);
+  });
+});
+
+describe("Edge case: files with only frontmatter", () => {
+  it("parseFrontmatter handles file with only frontmatter and empty body", () => {
+    const content = "---\ncreated: 2024-01-01\ntags: [test]\n---\n";
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).toBeDefined();
+    expect(frontmatter?.created).toBe("2024-01-01");
+    expect(body).toBe("");
+  });
+
+  it("chunkMarkdown handles frontmatter-only content", () => {
+    const content = "---\ncreated: 2024-01-01\n---\n";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    // Should produce at least one chunk containing the frontmatter text
+    expect(chunks.length).toBeGreaterThan(0);
+  });
+});
+
+describe("Edge case: Windows line endings", () => {
+  it("chunkMarkdown handles CRLF line endings", () => {
+    const content = "# Title\r\n\r\nParagraph one.\r\n\r\nParagraph two.\r\n";
+    const chunks = chunkMarkdown(content, { tokens: 256, overlap: 32 });
+    expect(chunks.length).toBeGreaterThan(0);
+    // Content should be preserved (with \r)
+    expect(chunks[0].text).toContain("Title");
+  });
+
+  it("parseFrontmatter handles CRLF frontmatter", () => {
+    const content = "---\r\ncreated: 2024-01-01\r\n---\r\n# Body";
+    // The regex uses \n, so CRLF may not match.
+    // This documents the current behavior.
+    const { frontmatter, body } = parseFrontmatter(content);
+    // On CRLF, the frontmatter regex may not match since it looks for \n
+    // This is a known limitation
+    if (frontmatter) {
+      expect(frontmatter.created).toBeDefined();
+    }
+    expect(body).toBeTruthy();
+  });
+});
+
+describe("Edge case: path validation", () => {
+  it("isMemoryPath rejects path traversal attempts", () => {
+    expect(isMemoryPath("../../../etc/passwd")).toBe(false);
+    expect(isMemoryPath("memory/../../etc/passwd")).toBe(true); // starts with memory/ so returns true
+  });
+
+  it("isMemoryPath handles various path formats", () => {
+    expect(isMemoryPath("MEMORY.md")).toBe(true);
+    expect(isMemoryPath("memory.md")).toBe(true);
+    expect(isMemoryPath("memory/notes.md")).toBe(true);
+    expect(isMemoryPath("memory/deep/nested/file.md")).toBe(true);
+    expect(isMemoryPath("MEMORY.txt")).toBe(false);
+    expect(isMemoryPath("notes.md")).toBe(false);
+    expect(isMemoryPath("src/memory.md")).toBe(false);
+  });
+
+  it("isMemoryPath handles empty and whitespace", () => {
+    expect(isMemoryPath("")).toBe(false);
+    expect(isMemoryPath("   ")).toBe(false);
+  });
+});
+
+describe("Edge case: cosineSimilarity", () => {
+  it("handles vectors of different lengths", () => {
+    const a = [1, 2, 3, 4, 5];
+    const b = [1, 2, 3];
+    // Should use min length
+    const sim = cosineSimilarity(a, b);
+    expect(sim).toBeGreaterThan(0);
+    expect(sim).toBeLessThanOrEqual(1);
+  });
+
+  it("handles very small values", () => {
+    const a = [1e-10, 1e-10];
+    const b = [1e-10, 1e-10];
+    const sim = cosineSimilarity(a, b);
+    expect(sim).toBeCloseTo(1);
+  });
+
+  it("handles large vectors", () => {
+    const size = 1536; // OpenAI embedding dimension
+    const a = Array.from({ length: size }, (_, i) => Math.sin(i));
+    const b = Array.from({ length: size }, (_, i) => Math.cos(i));
+    const sim = cosineSimilarity(a, b);
+    expect(sim).toBeGreaterThan(-1);
+    expect(sim).toBeLessThan(1);
+  });
+});
+
+describe("Edge case: session tracking", () => {
+  it("addSessionToContent handles content without existing frontmatter", () => {
+    const session: SessionContext = {
+      id: "test-123",
+      source: "test",
+    };
+    const result = addSessionToContent("# Hello", session);
+    expect(result).toContain("---");
+    expect(result).toContain("id: test-123");
+    expect(result).toContain("# Hello");
+  });
+
+  it("serializeFrontmatter handles empty session", () => {
+    const yaml = serializeFrontmatter({ session: {} });
+    expect(yaml).toContain("---");
+    expect(yaml).toContain("session:");
+  });
+
+  it("serializeFrontmatter handles paths with home directory", () => {
+    const home = os.homedir();
+    const session: SessionContext = {
+      project: path.join(home, "projects", "test"),
+    };
+    const yaml = serializeFrontmatter({ session });
+    // Should use ~ for home directory
+    expect(yaml).toContain("~/projects/test");
+  });
+
+  it("parseFrontmatter roundtrips through serialize", () => {
+    const original = {
+      session: { id: "abc", source: "test" },
+      created: "2024-01-01T00:00:00Z",
+      tags: ["tag1", "tag2"],
+    };
+    const serialized = serializeFrontmatter(original);
+    const { frontmatter } = parseFrontmatter(serialized + "body");
+
+    expect(frontmatter?.session?.id).toBe("abc");
+    expect(frontmatter?.session?.source).toBe("test");
+    expect(frontmatter?.created).toBe("2024-01-01T00:00:00Z");
+    expect(frontmatter?.tags).toEqual(["tag1", "tag2"]);
+  });
+});
+
+describe("Edge case: hashText determinism", () => {
+  it("same content always produces same hash", () => {
+    const content = "Hello, World!";
+    const hashes = Array.from({ length: 100 }, () => hashText(content));
+    const unique = new Set(hashes);
+    expect(unique.size).toBe(1);
+  });
+
+  it("different content produces different hashes", () => {
+    const hashes = new Set<string>();
+    for (let i = 0; i < 100; i++) {
+      hashes.add(hashText(`content-${i}`));
+    }
+    expect(hashes.size).toBe(100);
+  });
+
+  it("handles special characters consistently", () => {
+    const special = "line1\nline2\ttab\r\nwindows\0null";
+    const h1 = hashText(special);
+    const h2 = hashText(special);
+    expect(h1).toBe(h2);
+  });
+});
+
+describe("Edge case: buildFileEntry", () => {
+  let tempDir: string;
+
+  beforeEach(async () => {
+    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "minimem-edge-"));
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempDir, { recursive: true, force: true });
+  });
+
+  it("handles file with only whitespace", async () => {
+    const filePath = path.join(tempDir, "whitespace.md");
+    await fs.writeFile(filePath, "   \n\n   \n");
+
+    const entry = await buildFileEntry(filePath, tempDir);
+    expect(entry.size).toBeGreaterThan(0);
+    expect(entry.hash).toBe(hashText("   \n\n   \n"));
+  });
+
+  it("handles file with unicode content", async () => {
+    const content = "# 🚀 Launch Notes\n\n日本語テスト";
+    const filePath = path.join(tempDir, "unicode.md");
+    await fs.writeFile(filePath, content);
+
+    const entry = await buildFileEntry(filePath, tempDir);
+    expect(entry.hash).toBe(hashText(content));
+    expect(entry.size).toBeGreaterThan(0);
+  });
+
+  it("handles deeply nested path", async () => {
+    const nested = path.join(tempDir, "memory", "a", "b", "c");
+    await fs.mkdir(nested, { recursive: true });
+    const filePath = path.join(nested, "deep.md");
+    await fs.writeFile(filePath, "# Deep");
+
+    const entry = await buildFileEntry(filePath, tempDir);
+    expect(entry.path).toBe("memory/a/b/c/deep.md");
+  });
+});