contentbase 0.1.8 → 0.4.0

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "contentbase",
-  "version": "0.1.8",
+  "version": "0.4.0",
   "repository": "https://github.com/soederpop/contentbase",
   "website": "https://contentbase.soederpop.com",
   "type": "module",
-  "main": "./src/index.ts",
-  "types": "./src/index.ts",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "bin": {
     "cnotes": "./src/cli/index.ts",
     "contentbase": "./src/cli/index.ts"
@@ -13,6 +13,7 @@
   "scripts": {
     "test": "vitest run",
     "test:watch": "vitest",
+    "build": "find dist -name '*.d.ts' -o -name '*.d.ts.map' -o -name '*.js' -o -name '*.js.map' | xargs rm -f 2>/dev/null; tsc",
     "typecheck": "tsc --noEmit",
     "examples": "bun run scripts/examples/run-all.ts",
     "compile": "bun build ./src/cli/index.ts --compile --outfile dist/cnotes --external @node-llama-cpp/*",
@@ -20,7 +21,7 @@
     "release": "./scripts/release.sh"
   },
   "dependencies": {
-    "@soederpop/luca": ">=0.0.16",
+    "luca": ">= 3.0.2",
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.0",
     "mdast-util-mdxjs-esm": "^2.0.1",

package/src/adapters/node-fs.ts

@@ -0,0 +1,80 @@
+import fs from "fs/promises";
+import path from "path";
+import type { StorageAdapter, FileEntry, FileStat } from "../types";
+
+/**
+ * Default StorageAdapter that reads and writes from the local file system
+ * using Node's fs/promises module. This is used automatically when no
+ * adapter is passed to the Collection constructor.
+ */
+export class NodeStorageAdapter implements StorageAdapter {
+  async listFiles(rootPath: string, match: RegExp): Promise<FileEntry[]> {
+    const results: FileEntry[] = [];
+
+    const scan = async (dirPath: string) => {
+      let entries: string[];
+      try {
+        entries = await fs.readdir(dirPath);
+      } catch {
+        return;
+      }
+
+      for (const entry of entries) {
+        if (entry.startsWith(".")) continue;
+
+        const filePath = path.join(dirPath, entry);
+        const stat = await fs.stat(filePath);
+
+        if (stat.isDirectory()) {
+          await scan(filePath);
+        } else if (stat.isFile() && match.test(filePath)) {
+          results.push({
+            key: filePath,
+            stat: {
+              createdAt: stat.ctime,
+              updatedAt: stat.mtime,
+              size: stat.size,
+            },
+          });
+        }
+      }
+    };
+
+    await scan(rootPath);
+    return results;
+  }
+
+  async readFile(key: string): Promise<string> {
+    return fs.readFile(key, "utf8");
+  }
+
+  async stat(key: string): Promise<FileStat> {
+    const s = await fs.stat(key);
+    return { createdAt: s.ctime, updatedAt: s.mtime, size: s.size };
+  }
+
+  async writeFile(key: string, content: string): Promise<void> {
+    await fs.mkdir(path.parse(key).dir, { recursive: true });
+    await fs.writeFile(key, content, "utf8");
+  }
+
+  async deleteFile(key: string): Promise<void> {
+    try {
+      await fs.rm(key);
+    } catch {
+      // Silently ignore missing files — matches existing behavior
+    }
+  }
+
+  join(root: string, ...parts: string[]): string {
+    return path.resolve(root, ...parts);
+  }
+
+  relative(root: string, key: string): string {
+    return path.relative(root, key);
+  }
+
+  extname(key: string): string {
+    return path.extname(key);
+  }
+}

package/src/cli/commands/api/endpoints/doc.ts

@@ -173,8 +173,8 @@ ${html}
               if ('fetchAll' in rel) {
                 (result.relationships as any)[key] = rel.fetchAll().map((i: any) => ({ id: i.id, title: i.title }))
               } else if ('fetch' in rel) {
-                const parent = rel.fetch()
-                (result.relationships as any)[key] = parent ? { id: parent.id, title: parent.title } : null
+                const parentInstance = rel.fetch();
+                (result.relationships as any)[key] = parentInstance ? { id: parentInstance.id, title: parentInstance.title } : null
               }
             } catch {}
           }

package/src/cli/commands/api/endpoints/document.ts

@@ -58,8 +58,8 @@ export async function get(params: any, ctx: any) {
           if ('fetchAll' in rel) {
             (result.relationships as any)[key] = rel.fetchAll().map((i: any) => ({ id: i.id, title: i.title }))
           } else if ('fetch' in rel) {
-            const parent = rel.fetch()
-            (result.relationships as any)[key] = parent ? { id: parent.id, title: parent.title } : null
+            const parentInstance = rel.fetch();
+            (result.relationships as any)[key] = parentInstance ? { id: parentInstance.id, title: parentInstance.title } : null
           }
         } catch {}
       }

package/src/cli/commands/api/endpoints/search-reindex.ts

@@ -1,5 +1,6 @@
 import { z } from 'zod'
-import pathModule from 'node:path'
+import { collectDocumentInputs } from '../../../../search/document-inputs.js'
+import { getInitializedSemanticSearch } from '../../../../search/luca-semantic-search.js'
 
 export const path = '/api/search/reindex'
 export const description = 'Trigger search index rebuild'
@@ -14,14 +15,7 @@ export async function post(params: any, ctx: any) {
   const collection = ctx.container._contentbaseCollection
   const rootPath = collection.rootPath
 
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
-  if (!ctx.container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(ctx.container)
-  }
-
-  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
-  const ss = ctx.container.feature('semanticSearch', { dbPath })
-  await ss.initDb()
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
 
   if (params.pathIds) {
     await ss.reindex(params.pathIds)
@@ -30,23 +24,10 @@ export async function post(params: any, ctx: any) {
   }
 
   // Collect and re-index documents
-  const docs: any[] = []
   const targetIds = params.pathIds || collection.available
-
-  for (const pathId of targetIds) {
-    if (!collection.available.includes(pathId)) continue
-    const doc = collection.document(pathId)
-    const modelDef = collection.findModelDefinition(pathId)
-
-    docs.push({
-      pathId,
-      model: modelDef?.name ?? undefined,
-      title: doc.title,
-      slug: doc.slug,
-      meta: doc.meta,
-      content: doc.content,
-    })
-  }
+  const targetSet = new Set(targetIds)
+  const docs = collectDocumentInputs(collection)
+    .filter((doc) => targetSet.has(doc.pathId))
 
   const toIndex = params.force ? docs : docs.filter((d: any) => ss.needsReindex(d))
 

package/src/cli/commands/api/endpoints/search-status.ts

@@ -1,6 +1,5 @@
 import { z } from 'zod'
-import pathModule from 'node:path'
-import { existsSync, readdirSync } from 'node:fs'
+import { getInitializedSemanticSearch, hasSearchIndex } from '../../../../search/luca-semantic-search.js'
 
 export const path = '/api/search/status'
 export const description = 'Search index health and statistics'
@@ -11,16 +10,7 @@ export const getSchema = z.object({})
 export async function get(_params: any, ctx: any) {
   const collection = ctx.container._contentbaseCollection
   const rootPath = collection.rootPath
-  const dbDir = pathModule.join(rootPath, '.contentbase')
-
-  const hasIndex = existsSync(dbDir) && (() => {
-    try {
-      const files = readdirSync(dbDir) as string[]
-      return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-    } catch {
-      return false
-    }
-  })()
+  const hasIndex = hasSearchIndex(rootPath)
 
   if (!hasIndex) {
     return {
@@ -37,14 +27,7 @@ export async function get(_params: any, ctx: any) {
     }
   }
 
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
-  if (!ctx.container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(ctx.container)
-  }
-
-  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
-  const ss = ctx.container.feature('semanticSearch', { dbPath })
-  await ss.initDb()
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
 
   const stats = ss.getStats()
   return {

package/src/cli/commands/api/endpoints/search.ts

@@ -1,40 +1,10 @@
 import { z } from 'zod'
-import pathModule from 'node:path'
-import { existsSync, readdirSync } from 'node:fs'
+import { getInitializedSemanticSearch, hasSearchIndex } from '../../../../search/luca-semantic-search.js'
 
 export const path = '/api/search'
 export const description = 'Search across collection documents using keyword, semantic, or hybrid modes'
 export const tags = ['query']
 
-// ── Helpers ──────────────────────────────────────────────────────────
-
-function hasSearchIndex(rootPath: string): boolean {
-  const dbDir = pathModule.join(rootPath, '.contentbase')
-  if (!existsSync(dbDir)) return false
-  try {
-    const files = readdirSync(dbDir) as string[]
-    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-  } catch {
-    return false
-  }
-}
-
-let _semanticSearch: any = null
-
-async function getSemanticSearch(container: any, rootPath: string) {
-  if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
-
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
-  if (!container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(container)
-  }
-
-  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
-  _semanticSearch = container.feature('semanticSearch', { dbPath })
-  await _semanticSearch.initDb()
-  return _semanticSearch
-}
-
 async function doSearch(ss: any, query: string, mode: string, options: any) {
   switch (mode) {
     case 'keyword':
@@ -65,7 +35,7 @@ export async function get(params: any, ctx: any) {
     return { error: 'No search index found. Run: cnotes embed' }
   }
 
-  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
   const mode = params.mode || 'hybrid'
   const limit = params.limit ? parseInt(params.limit, 10) : 10
   const searchOptions = { limit, model: params.model }
@@ -92,7 +62,7 @@ export async function post(params: any, ctx: any) {
     return { error: 'No search index found. Run: cnotes embed' }
   }
 
-  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
   const mode = params.mode || 'hybrid'
   const searchOptions = {
     limit: params.limit || 10,

package/src/cli/commands/api/endpoints/semantic-search.ts

@@ -1,40 +1,10 @@
 import { z } from 'zod'
-import pathModule from 'node:path'
-import { existsSync, readdirSync } from 'node:fs'
+import { getInitializedSemanticSearch, hasSearchIndex } from '../../../../search/luca-semantic-search.js'
 
 export const path = '/api/semantic-search'
 export const description = 'Semantic search across collection documents'
 export const tags = ['query']
 
-// ── Helpers ──────────────────────────────────────────────────────────
-
-function hasSearchIndex(rootPath: string): boolean {
-  const dbDir = pathModule.join(rootPath, '.contentbase')
-  if (!existsSync(dbDir)) return false
-  try {
-    const files = readdirSync(dbDir) as string[]
-    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-  } catch {
-    return false
-  }
-}
-
-let _semanticSearch: any = null
-
-async function getSemanticSearch(container: any, rootPath: string) {
-  if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
-
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
-  if (!container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(container)
-  }
-
-  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
-  _semanticSearch = container.feature('semanticSearch', { dbPath })
-  await _semanticSearch.initDb()
-  return _semanticSearch
-}
-
 // ── GET /api/semantic-search ─────────────────────────────────────────
 
 export const getSchema = z.object({
@@ -53,7 +23,7 @@ export async function get(params: any, ctx: any) {
     return { error: 'No search index found. Run: cnotes embed' }
   }
 
-  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
   const mode = params.mode || 'hybrid'
   const limit = params.limit ? parseInt(params.limit, 10) : 10
   const searchOptions = { limit, model: params.model }
@@ -94,7 +64,7 @@ export async function post(params: any, ctx: any) {
     return { error: 'No search index found. Run: cnotes embed' }
   }
 
-  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const ss = await getInitializedSemanticSearch(ctx.container, rootPath)
   const mode = params.mode || 'hybrid'
   const searchOptions = {
     limit: params.limit || 10,

package/src/cli/commands/embed.ts

@@ -1,8 +1,10 @@
 import { z } from 'zod'
-import { existsSync, readdirSync, accessSync, constants as fsConstants } from 'node:fs'
+import { existsSync, accessSync, constants as fsConstants } from 'node:fs'
 import path from 'node:path'
 import { commands } from '../registry.js'
 import { loadCollection } from '../load-collection.js'
+import { collectDocumentInputs } from '../../search/document-inputs.js'
+import { createSemanticSearch, ensureSemanticSearchAttached, getInitializedSemanticSearch, hasSearchIndex, loadSemanticSearchClass } from '../../search/luca-semantic-search.js'
 
 const argsSchema = z.object({
   force: z.boolean().default(false),
@@ -13,67 +15,6 @@ const argsSchema = z.object({
   contentFolder: z.string().optional(),
 })
 
-function hasSearchIndex(rootPath: string): boolean {
-  const dbDir = path.join(rootPath, '.contentbase')
-  if (!existsSync(dbDir)) return false
-  try {
-    const files = readdirSync(dbDir) as string[]
-    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-  } catch {
-    return false
-  }
-}
-
-function collectDocumentInputs(collection: any) {
-  const inputs: any[] = []
-  for (const pathId of collection.available) {
-    const doc = collection.document(pathId)
-    const modelDef = collection.findModelDefinition(pathId)
-
-    const sections: any[] = []
-    const lines = (doc.content as string).split('\n')
-    let currentHeading: string | null = null
-    let currentContent: string[] = []
-
-    for (const line of lines) {
-      const h2Match = line.match(/^## (.+)/)
-      if (h2Match) {
-        if (currentHeading) {
-          sections.push({
-            heading: currentHeading,
-            headingPath: currentHeading,
-            content: currentContent.join('\n').trim(),
-            level: 2,
-          })
-        }
-        currentHeading = h2Match[1].trim()
-        currentContent = []
-      } else if (currentHeading) {
-        currentContent.push(line)
-      }
-    }
-    if (currentHeading) {
-      sections.push({
-        heading: currentHeading,
-        headingPath: currentHeading,
-        content: currentContent.join('\n').trim(),
-        level: 2,
-      })
-    }
-
-    inputs.push({
-      pathId,
-      model: modelDef?.name ?? undefined,
-      title: doc.title,
-      slug: doc.slug,
-      meta: doc.meta,
-      content: doc.content,
-      sections: sections.length > 0 ? sections : undefined,
-    })
-  }
-  return inputs
-}
-
 function isLocalInstalled(): boolean {
   const modulePath = path.join(process.cwd(), 'node_modules', 'node-llama-cpp')
   try {
@@ -103,10 +44,8 @@ async function installLocal(SemanticSearch: any, container: any): Promise<void>
   console.error(`Installing node-llama-cpp@${SemanticSearch.PINNED_LLAMA_VERSION}...`)
 
   // Create a temporary instance to use the install method
-  if (!container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(container)
-  }
-  const ss = container.feature('semanticSearch', {
+  await ensureSemanticSearchAttached(container, SemanticSearch)
+  const ss = await createSemanticSearch(container, cwd, {
     dbPath: path.join(cwd, '.contentbase/search.sqlite'),
     embeddingProvider: 'local',
   })
@@ -138,7 +77,7 @@ function detectInstallCommand(cwd: string, version: string): string {
 }
 
 async function handler(options: z.infer<typeof argsSchema>, { container }: { container: any }) {
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  const SemanticSearch = await loadSemanticSearchClass()
 
   // --install-local: install node-llama-cpp only, then exit
   if (options.installLocal) {
@@ -171,15 +110,9 @@ async function handler(options: z.infer<typeof argsSchema>, { container }: { con
       return
     }
 
-    if (!container.features.available.includes('semanticSearch')) {
-      SemanticSearch.attach(container)
-    }
-    const dbPath = path.join(collection.rootPath, '.contentbase/search.sqlite')
-    const ss = container.feature('semanticSearch', {
-      dbPath,
+    const ss = await getInitializedSemanticSearch(container, collection.rootPath, {
       embeddingProvider: provider,
     })
-    await ss.initDb()
 
     const stats = ss.getStats()
     console.log('Search Index Status')
@@ -206,15 +139,9 @@ async function handler(options: z.infer<typeof argsSchema>, { container }: { con
   }
 
   // Embed / re-embed
-  if (!container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(container)
-  }
-  const dbPath = path.join(collection.rootPath, '.contentbase/search.sqlite')
-  const ss = container.feature('semanticSearch', {
-    dbPath,
+  const ss = await getInitializedSemanticSearch(container, collection.rootPath, {
     embeddingProvider: provider,
   })
-  await ss.initDb()
 
   const docs = collectDocumentInputs(collection)
   const startTime = Date.now()