contentbase 0.0.2 → 0.0.4

package/showcases/node_modules/.cache/.repl_history

@@ -0,0 +1,3 @@
+luca.networing
+luca.networking
+container.feature('networking')

package/showcases/vinyl-collection/artists/nina-simone.mdx

@@ -1,7 +1,7 @@
 ---
 genre: Jazz / Soul
 origin: Tryon, North Carolina
-active: "1954–2003"
+active: 1954–2003
 ---
 
 # Nina Simone

package/src/__tests__/semantic-search.integration.test.ts

@@ -0,0 +1,284 @@
+/**
+ * Semantic Search Integration Tests
+ *
+ * Run with: bun test src/__tests__/semantic-search.integration.test.ts
+ *
+ * Requires:
+ * - bun runtime (uses bun:sqlite)
+ * - OPENAI_API_KEY env var for embedding tests
+ *
+ * Tests the full pipeline: collection → index → search → results
+ */
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import path from "path";
+import fs from "fs/promises";
+import { existsSync } from "fs";
+import { Collection } from "../../src/collection";
+
+const dir = import.meta.dirname ?? new URL(".", import.meta.url).pathname;
+const FIXTURES_PATH = path.resolve(dir, "../../test/fixtures/sdlc");
+const DB_DIR = path.join(FIXTURES_PATH, ".contentbase-test");
+const DB_PATH = path.join(DB_DIR, "search.sqlite");
+
+const HAS_API_KEY = !!process.env.OPENAI_API_KEY;
+
+function collectDocumentInputs(collection: Collection) {
+  const inputs: any[] = [];
+  for (const pathId of collection.available) {
+    const doc = collection.document(pathId);
+    const modelDef = (collection as any).findModelDefinition?.(pathId);
+
+    const sections: any[] = [];
+    const lines = doc.content.split("\n");
+    let currentHeading: string | null = null;
+    let currentContent: string[] = [];
+
+    for (const line of lines) {
+      const h2Match = line.match(/^## (.+)/);
+      if (h2Match) {
+        if (currentHeading) {
+          sections.push({
+            heading: currentHeading,
+            headingPath: currentHeading,
+            content: currentContent.join("\n").trim(),
+            level: 2,
+          });
+        }
+        currentHeading = h2Match[1].trim();
+        currentContent = [];
+      } else if (currentHeading) {
+        currentContent.push(line);
+      }
+    }
+    if (currentHeading) {
+      sections.push({
+        heading: currentHeading,
+        headingPath: currentHeading,
+        content: currentContent.join("\n").trim(),
+        level: 2,
+      });
+    }
+
+    inputs.push({
+      pathId,
+      model: modelDef?.name ?? undefined,
+      title: doc.title,
+      slug: (doc as any).slug,
+      meta: doc.meta,
+      content: doc.content,
+      sections: sections.length > 0 ? sections : undefined,
+    });
+  }
+  return inputs;
+}
+
+describe("Semantic Search Integration", () => {
+  let collection: Collection;
+  let SemanticSearchClass: any;
+  let ss: any;
+
+  beforeAll(async () => {
+    // Import models
+    const { Epic, Story } = await import("../../test/fixtures/sdlc/models");
+
+    // Load collection
+    collection = new Collection({ rootPath: FIXTURES_PATH, name: "test-sdlc" });
+    collection.register(Epic);
+    collection.register(Story);
+    await collection.load();
+
+    // Import SemanticSearch
+    const mod = await import("@soederpop/luca/agi");
+    SemanticSearchClass = mod.SemanticSearch;
+
+    // Clean up any previous test index
+    if (existsSync(DB_DIR)) {
+      await fs.rm(DB_DIR, { recursive: true });
+    }
+  });
+
+  afterAll(async () => {
+    if (ss) {
+      try { await ss.close(); } catch {}
+    }
+    if (existsSync(DB_DIR)) {
+      await fs.rm(DB_DIR, { recursive: true });
+    }
+  });
+
+  describe("Database Layer", () => {
+    it("creates and initializes database tables", async () => {
+      // Use the class directly - create an instance through the static attach pattern
+      // We need a minimal container mock for the Feature constructor
+      const { Database } = await import("bun:sqlite");
+
+      // Clean first
+      if (existsSync(DB_DIR)) await fs.rm(DB_DIR, { recursive: true });
+
+      // Test the Database creation directly using bun:sqlite
+      const { mkdirSync } = await import("fs");
+      mkdirSync(DB_DIR, { recursive: true });
+      const dbPath = path.join(DB_DIR, "search.openai-text-embedding-3-small.sqlite");
+      const db = new Database(dbPath);
+
+      db.exec("PRAGMA journal_mode = WAL");
+      db.exec("PRAGMA foreign_keys = ON");
+
+      // Create the same tables as SemanticSearch
+      db.exec(`CREATE TABLE search_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
+      db.exec(`CREATE TABLE documents (
+        path_id TEXT PRIMARY KEY, model TEXT, title TEXT, slug TEXT,
+        meta_json TEXT, content TEXT, sections_json TEXT,
+        content_hash TEXT, indexed_at TEXT
+      )`);
+      db.exec(`CREATE VIRTUAL TABLE documents_fts USING fts5(
+        path_id, title, content, sections_text, tokenize='porter unicode61'
+      )`);
+      db.exec(`CREATE TABLE chunks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        path_id TEXT NOT NULL, section TEXT, heading_path TEXT,
+        seq INTEGER NOT NULL, content TEXT NOT NULL,
+        content_hash TEXT NOT NULL, embedding BLOB,
+        FOREIGN KEY (path_id) REFERENCES documents(path_id) ON DELETE CASCADE
+      )`);
+
+      // Verify tables exist
+      const tables = db.query("SELECT name FROM sqlite_master WHERE type='table'").all() as any[];
+      const tableNames = tables.map((t: any) => t.name);
+      expect(tableNames).toContain("search_meta");
+      expect(tableNames).toContain("documents");
+      expect(tableNames).toContain("chunks");
+
+      db.close();
+      // Clean up for the real test
+      await fs.rm(DB_DIR, { recursive: true });
+    });
+  });
+
+  describe("Document Chunking", () => {
+    it("collectDocumentInputs extracts sections from fixture documents", () => {
+      const docs = collectDocumentInputs(collection);
+      expect(docs.length).toBeGreaterThan(0);
+
+      const authEpic = docs.find((d: any) => d.pathId === "epics/authentication");
+      expect(authEpic).toBeDefined();
+      expect(authEpic.title).toBe("Authentication");
+      expect(authEpic.model).toBe("Epic");
+      expect(authEpic.sections).toBeDefined();
+      expect(authEpic.sections.length).toBeGreaterThan(0);
+    });
+
+    it("documents without h2 sections get no sections array", () => {
+      const docs = collectDocumentInputs(collection);
+      // Stories may or may not have h2 sections depending on fixture
+      for (const doc of docs) {
+        if (doc.sections) {
+          expect(doc.sections.length).toBeGreaterThan(0);
+          for (const section of doc.sections) {
+            expect(section).toHaveProperty("heading");
+            expect(section).toHaveProperty("content");
+            expect(section).toHaveProperty("level");
+          }
+        }
+      }
+    });
+  });
+
+  describe("Full Pipeline (requires OPENAI_API_KEY)", () => {
+    it.skipIf(!HAS_API_KEY)("indexes documents and generates embeddings", async () => {
+      // Create a container-like environment for SemanticSearch
+      const container = (await import("@soederpop/luca")).default;
+
+      if (!container.features.available.includes("semanticSearch")) {
+        SemanticSearchClass.attach(container);
+      }
+
+      ss = container.feature("semanticSearch", {
+        dbPath: DB_PATH,
+        embeddingProvider: "openai",
+        embeddingModel: "text-embedding-3-small",
+      });
+
+      await ss.initDb();
+      expect(ss.state.get("dbReady")).toBe(true);
+
+      const docs = collectDocumentInputs(collection);
+      await ss.indexDocuments(docs);
+
+      const stats = ss.getStats();
+      expect(stats.documentCount).toBe(docs.length);
+      expect(stats.chunkCount).toBeGreaterThan(0);
+      expect(stats.embeddingCount).toBeGreaterThan(0);
+      expect(stats.provider).toBe("openai");
+      expect(stats.model).toBe("text-embedding-3-small");
+      expect(stats.dimensions).toBe(1536);
+    }, 60000);
+
+    it.skipIf(!HAS_API_KEY)("keyword search returns BM25-ranked results", async () => {
+      const results = await ss.search("authentication");
+      expect(results.length).toBeGreaterThan(0);
+      expect(results[0].score).toBeGreaterThan(0);
+      expect(results[0].snippet).toBeTruthy();
+    });
+
+    it.skipIf(!HAS_API_KEY)("vector search finds semantically related docs", async () => {
+      const results = await ss.vectorSearch("user login and registration");
+      expect(results.length).toBeGreaterThan(0);
+      const authResult = results.find((r: any) => r.pathId.includes("authentication"));
+      expect(authResult).toBeDefined();
+    });
+
+    it.skipIf(!HAS_API_KEY)("hybrid search combines both modes", async () => {
+      const results = await ss.hybridSearch("authentication login");
+      expect(results.length).toBeGreaterThan(0);
+      expect(results[0].score).toBeGreaterThan(0);
+    });
+
+    it.skipIf(!HAS_API_KEY)("model filter restricts results", async () => {
+      const epicResults = await ss.search("authentication", { model: "Epic" });
+      for (const r of epicResults) {
+        expect(r.model).toBe("Epic");
+      }
+    });
+
+    it.skipIf(!HAS_API_KEY)("search results include citation fields", async () => {
+      const results = await ss.hybridSearch("authentication");
+      expect(results.length).toBeGreaterThan(0);
+      const r = results[0];
+      expect(r).toHaveProperty("pathId");
+      expect(r).toHaveProperty("model");
+      expect(r).toHaveProperty("title");
+      expect(r).toHaveProperty("score");
+      expect(r).toHaveProperty("snippet");
+    });
+
+    it.skipIf(!HAS_API_KEY)("needsReindex returns false for unchanged docs", async () => {
+      const docs = collectDocumentInputs(collection);
+      for (const doc of docs) {
+        expect(ss.needsReindex(doc)).toBe(false);
+      }
+    });
+
+    it.skipIf(!HAS_API_KEY)("search with no index throws actionable error", async () => {
+      // Create a fresh instance with different dbPath
+      const container = (await import("@soederpop/luca")).default;
+      const freshSs = container.feature("semanticSearch", {
+        dbPath: path.join(DB_DIR, "nonexistent.sqlite"),
+        embeddingProvider: "openai",
+      });
+
+      // Searching without initDb should throw
+      expect(() => freshSs.search("test")).toThrow();
+    });
+
+    it.skipIf(!HAS_API_KEY)("getStats returns correct index status", async () => {
+      const stats = ss.getStats();
+      expect(stats.documentCount).toBeGreaterThan(0);
+      expect(stats.chunkCount).toBeGreaterThan(0);
+      expect(stats.embeddingCount).toBe(stats.chunkCount);
+      expect(stats.lastIndexedAt).toBeTruthy();
+      expect(stats.dimensions).toBe(1536);
+      expect(stats.dbSizeBytes).toBeGreaterThan(0);
+    });
+  });
+});

package/src/api/endpoints/actions.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod'
+
+export const path = '/api/actions'
+export const description = 'List or execute collection actions'
+export const tags = ['actions']
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  return { actions: collection.availableActions }
+}
+
+export const postSchema = z.object({
+  name: z.string(),
+  args: z.array(z.any()).optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  if (ctx.container._contentbaseReadOnly) {
+    ctx.response.status(403)
+    return { error: 'Server is running in read-only mode' }
+  }
+
+  const collection = ctx.container._contentbaseCollection
+
+  if (!collection.availableActions.includes(params.name)) {
+    ctx.response.status(400)
+    return {
+      error: `Unknown action: ${params.name}. Available: ${collection.availableActions.join(', ') || '(none)'}`,
+    }
+  }
+
+  const result = await collection.runAction(params.name, ...(params.args || []))
+  return typeof result === 'string' ? { result } : result
+}

package/src/api/endpoints/doc.ts

@@ -0,0 +1,187 @@
+import matter from 'gray-matter'
+import { renderMarkdownToHtml, rewriteDocLinks } from '../helpers.js'
+
+export const path = '/docs/:docPath(.*)'
+export const description = 'Content-negotiated document serving (JSON, HTML, or Markdown)'
+export const tags = ['docs']
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  let docPath: string = params.docPath || ''
+
+  // Determine format from extension or Accept header
+  let format = 'json'
+  if (docPath.endsWith('.json')) {
+    format = 'json'
+    docPath = docPath.slice(0, -5)
+  } else if (docPath.endsWith('.html')) {
+    format = 'html'
+    docPath = docPath.slice(0, -5)
+  } else if (docPath.endsWith('.md')) {
+    format = 'md'
+    docPath = docPath.slice(0, -3)
+  } else {
+    const accept = ctx.request.headers?.accept || ''
+    if (accept.includes('text/html')) format = 'html'
+    else if (accept.includes('text/markdown')) format = 'md'
+  }
+
+  if (!collection.available.includes(docPath)) {
+    ctx.response.status(404)
+    return { error: `Document not found: ${docPath}` }
+  }
+
+  const doc = collection.document(docPath)
+  const modelDef = collection.findModelDefinition(docPath)
+
+  switch (format) {
+    case 'md': {
+      const raw = matter.stringify(doc.content, doc.meta)
+      ctx.response.type('text/markdown')
+      ctx.response.send(raw)
+      return
+    }
+    case 'html': {
+      const html = rewriteDocLinks(await renderMarkdownToHtml(doc.content))
+      const page = `<!DOCTYPE html>
+<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${doc.title}</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600&family=Inter:wght@400;500;600&display=swap" rel="stylesheet">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"><\/script>
+<style>
+  *, *::before, *::after { box-sizing: border-box; }
+  body {
+    font-family: 'Inter', system-ui, -apple-system, sans-serif;
+    max-width: 52rem;
+    margin: 0 auto;
+    padding: 2rem 1.5rem;
+    line-height: 1.7;
+    color: #1a1a2e;
+    background: #fafafa;
+  }
+  h1 { font-size: 2rem; font-weight: 600; margin: 2rem 0 1rem; color: #0f0f23; }
+  h2 { font-size: 1.5rem; font-weight: 600; margin: 2.5rem 0 0.75rem; color: #16163a; border-bottom: 1px solid #e2e2e8; padding-bottom: 0.4rem; }
+  h3 { font-size: 1.2rem; font-weight: 600; margin: 2rem 0 0.5rem; color: #1a1a2e; }
+  h4, h5, h6 { font-weight: 600; margin: 1.5rem 0 0.5rem; }
+  p { margin: 0 0 1rem; }
+  a { color: #2563eb; text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  blockquote {
+    border-left: 3px solid #6366f1;
+    margin: 1rem 0;
+    padding: 0.5rem 1rem;
+    background: #f0f0ff;
+    color: #3730a3;
+    border-radius: 0 6px 6px 0;
+  }
+  blockquote p { margin: 0; }
+  code {
+    font-family: 'Fira Code', 'Cascadia Code', 'JetBrains Mono', monospace;
+    font-feature-settings: "liga" 1, "calt" 1;
+    -webkit-font-feature-settings: "liga" 1, "calt" 1;
+    font-size: 0.875em;
+    background: #ededf0;
+    padding: 0.15em 0.4em;
+    border-radius: 4px;
+    color: #d6336c;
+  }
+  pre {
+    margin: 1rem 0;
+    border-radius: 8px;
+    overflow: hidden;
+    background: #282c34;
+    box-shadow: 0 2px 8px rgba(0,0,0,0.12);
+  }
+  pre code {
+    display: block;
+    padding: 1.25rem 1.5rem;
+    overflow-x: auto;
+    font-size: 0.9rem;
+    line-height: 1.6;
+    background: none;
+    color: #abb2bf;
+    border-radius: 0;
+  }
+  pre code .hljs-comment { font-style: italic; }
+  ul, ol { padding-left: 1.5rem; margin: 0.5rem 0 1rem; }
+  li { margin: 0.3rem 0; }
+  li > ul, li > ol { margin: 0.2rem 0; }
+  table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
+  th { background: #f0f0f5; font-weight: 600; text-align: left; }
+  td, th { border: 1px solid #ddd; padding: 0.6em 0.8em; font-size: 0.95rem; }
+  tr:nth-child(even) { background: #f8f8fb; }
+  hr { border: none; border-top: 1px solid #e2e2e8; margin: 2rem 0; }
+  img { max-width: 100%; height: auto; border-radius: 6px; }
+  .meta-header { color: #6b7280; font-size: 0.85rem; margin-bottom: 2rem; padding-bottom: 1rem; border-bottom: 1px solid #e2e2e8; }
+  .meta-header span { margin-right: 1.5rem; }
+</style>
+</head>
+<body>
+${doc.meta && Object.keys(doc.meta).length > 0 ? `<div class="meta-header">${Object.entries(doc.meta).filter(([k]) => k !== 'title').map(([k, v]) => `<span><strong>${k}:</strong> ${v}</span>`).join('')}</div>` : ''}
+${html}
+<script>hljs.highlightAll();<\/script>
+</body></html>`
+      ctx.response.type('text/html')
+      ctx.response.send(page)
+      return
+    }
+    default: {
+      const result: Record<string, unknown> = {
+        id: doc.id,
+        title: doc.title,
+        meta: doc.meta,
+        content: doc.content,
+        outline: doc.toOutline(),
+        model: modelDef?.name || null,
+        createdAt: doc.createdAt,
+        updatedAt: doc.updatedAt,
+        size: doc.size,
+      }
+
+      if (modelDef) {
+        const instance = collection.getModel(docPath, modelDef)
+        const sectionKeys = modelDef.sections ? Object.keys(modelDef.sections) : []
+        const computedKeys = modelDef.computed ? Object.keys(modelDef.computed) : []
+        const relationshipKeys = modelDef.relationships ? Object.keys(modelDef.relationships) : []
+
+        if (sectionKeys.length) {
+          result.sections = {}
+          for (const key of sectionKeys) {
+            try {
+              (result.sections as any)[key] = instance.sections[key]
+            } catch {}
+          }
+        }
+
+        if (computedKeys.length) {
+          result.computed = {}
+          for (const key of computedKeys) {
+            try {
+              (result.computed as any)[key] = instance.computed[key]
+            } catch {}
+          }
+        }
+
+        if (relationshipKeys.length) {
+          result.relationships = {}
+          for (const key of relationshipKeys) {
+            try {
+              const rel = (instance.relationships as any)[key]
+              if ('fetchAll' in rel) {
+                (result.relationships as any)[key] = rel.fetchAll().map((i: any) => ({ id: i.id, title: i.title }))
+              } else if ('fetch' in rel) {
+                const parent = rel.fetch()
+                (result.relationships as any)[key] = parent ? { id: parent.id, title: parent.title } : null
+              }
+            } catch {}
+          }
+        }
+      }
+
+      return result
+    }
+  }
+}

package/src/api/endpoints/docs-index.ts

@@ -0,0 +1,26 @@
+import { renderTocPage } from '../helpers.js'
+
+export const path = '/docs'
+export const description = 'Table of contents for all documents'
+export const tags = ['docs']
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const accept = ctx.request.headers?.accept || ''
+
+  if (accept.includes('text/html') || !accept.includes('application/json')) {
+    const page = await renderTocPage(collection)
+    ctx.response.type('text/html')
+    ctx.response.send(page)
+    return
+  }
+
+  return {
+    title: 'Table of Contents',
+    documents: collection.available.map((id: string) => {
+      const doc = collection.document(id)
+      const modelDef = collection.findModelDefinition(id)
+      return { id, title: doc.title, model: modelDef?.name || null }
+    })
+  }
+}