contentbase 0.1.8 → 0.4.0

package/src/cli/commands/search.ts

@@ -1,8 +1,9 @@
 import { z } from 'zod'
-import { existsSync, readdirSync } from 'node:fs'
 import path from 'node:path'
 import { commands } from '../registry.js'
 import { loadCollection } from '../load-collection.js'
+import { collectDocumentInputs } from '../../search/document-inputs.js'
+import { getInitializedSemanticSearch, hasSearchIndex } from '../../search/luca-semantic-search.js'
 
 const argsSchema = z.object({
   mode: z.enum(['hybrid', 'keyword', 'vector']).default('hybrid'),
@@ -15,31 +16,8 @@ const argsSchema = z.object({
   contentFolder: z.string().optional(),
 })
 
-function hasSearchIndex(rootPath: string): boolean {
-  const dbDir = path.join(rootPath, '.contentbase')
-  if (!existsSync(dbDir)) return false
-  try {
-    const files = readdirSync(dbDir) as string[]
-    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-  } catch {
-    return false
-  }
-}
-
-async function getSemanticSearch(container: any, rootPath: string) {
-  const { SemanticSearch } = await import('@soederpop/luca/agi')
-  if (!container.features.available.includes('semanticSearch')) {
-    SemanticSearch.attach(container)
-  }
-
-  const dbPath = path.join(rootPath, '.contentbase/search.sqlite')
-  const ss = container.feature('semanticSearch', { dbPath })
-  await ss.initDb()
-  return ss
-}
-
 async function buildIndex(container: any, collection: any) {
-  const ss = await getSemanticSearch(container, collection.rootPath)
+  const ss = await getInitializedSemanticSearch(container, collection.rootPath)
   const docs = collectDocumentInputs(collection)
 
   const toIndex = docs.filter((doc: any) => ss.needsReindex(doc))
@@ -61,56 +39,6 @@ async function buildIndex(container: any, collection: any) {
   return ss
 }
 
-function collectDocumentInputs(collection: any) {
-  const inputs: any[] = []
-  for (const pathId of collection.available) {
-    const doc = collection.document(pathId)
-    const modelDef = collection.findModelDefinition(pathId)
-
-    const sections: any[] = []
-    const lines = (doc.content as string).split('\n')
-    let currentHeading: string | null = null
-    let currentContent: string[] = []
-
-    for (const line of lines) {
-      const h2Match = line.match(/^## (.+)/)
-      if (h2Match) {
-        if (currentHeading) {
-          sections.push({
-            heading: currentHeading,
-            headingPath: currentHeading,
-            content: currentContent.join('\n').trim(),
-            level: 2,
-          })
-        }
-        currentHeading = h2Match[1].trim()
-        currentContent = []
-      } else if (currentHeading) {
-        currentContent.push(line)
-      }
-    }
-    if (currentHeading) {
-      sections.push({
-        heading: currentHeading,
-        headingPath: currentHeading,
-        content: currentContent.join('\n').trim(),
-        level: 2,
-      })
-    }
-
-    inputs.push({
-      pathId,
-      model: modelDef?.name ?? undefined,
-      title: doc.title,
-      slug: doc.slug,
-      meta: doc.meta,
-      content: doc.content,
-      sections: sections.length > 0 ? sections : undefined,
-    })
-  }
-  return inputs
-}
-
 async function handler(options: z.infer<typeof argsSchema>, { container }: { container: any }) {
   const ui = container.feature('ui')
   const query = container.argv._[1] as string | undefined
@@ -140,7 +68,7 @@ async function handler(options: z.infer<typeof argsSchema>, { container }: { con
   if (options.bootstrap && !hasSearchIndex(collection.rootPath)) {
     ss = await buildIndex(container, collection)
   } else {
-    ss = await getSemanticSearch(container, collection.rootPath)
+    ss = await getInitializedSemanticSearch(container, collection.rootPath)
   }
 
   // Parse where clause: "key=value,key2=value2"
@@ -193,22 +121,29 @@ async function handler(options: z.infer<typeof argsSchema>, { container }: { con
     return
   }
 
-  const header = `| # | Score | Document | Match |`
-  const separator = `|---|-------|----------|-------|`
-  const rows = results.map((r: any, i: number) => {
-    const score = `${Math.round(r.score * 100)}%`
-    const doc = r.title || r.pathId
-    let match = ''
+  const colors = ui.colors
+  const cols = process.stdout.columns || 80
+  const pad = '   '
+  const maxSnippet = cols - pad.length - 2
+
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i]
+    const title = r.title || r.pathId
+    console.log(`${colors.dim(`${i + 1}.`)} ${colors.bold(title)}`)
+    let relPath = r.pathId
+    try { relPath = path.relative(process.cwd(), collection.document(r.pathId).path) } catch {}
+    console.log(`${pad}${colors.cyan(relPath)}`)
     if (r.snippet) {
-      const snippet = r.snippet.replace(/\n/g, ' ').replace(/\|/g, '\\|').substring(0, 80)
-      const section = r.matchedSection || ''
-      match = section ? `**${section}**: ${snippet}` : snippet
+      let snippet = r.snippet
+        .replace(/>>>/g, '').replace(/<<</g, '')
+        .replace(/\n/g, ' ').replace(/\s+/g, ' ')
+        .replace(/[`*_~\[\]]/g, '').trim()
+      if (r.matchedSection) snippet = `${r.matchedSection} — ${snippet}`
+      if (snippet.length > maxSnippet) snippet = snippet.substring(0, maxSnippet - 1) + '…'
+      console.log(`${pad}${colors.dim(snippet)}`)
     }
-    return `| ${i + 1} | ${score} | ${doc} | ${match} |`
-  })
-
-  const table = [header, separator, ...rows].join('\n')
-  ui.markdown(table)
+    if (i < results.length - 1) console.log()
+  }
 
   if (options.full) {
     for (const r of results) {

package/src/cli/commands/serve.ts

@@ -5,6 +5,8 @@ import { commands } from '../registry.js'
 import { loadCollection } from '../load-collection.js'
 import { buildSchemaJSON } from './api/helpers.js'
 import { builtinEndpoints } from './api/endpoints/index.js'
+import { collectDocumentInputs } from '../../search/document-inputs.js'
+import { getInitializedSemanticSearch, hasSearchIndex } from '../../search/luca-semantic-search.js'
 
 const argsSchema = z.object({
   port: z.number().default(8000),
@@ -184,39 +186,12 @@ async function handler(options: z.infer<typeof argsSchema>, context: { container
   // Search index auto-update
   // ---------------------------------------------------------------------------
   if (options.search) {
-    const { existsSync, readdirSync } = await import('node:fs')
-    const dbDir = path.join(collection.rootPath, '.contentbase')
-    const hasIndex = existsSync(dbDir) && (() => {
+    if (hasSearchIndex(collection.rootPath)) {
       try {
-        return (readdirSync(dbDir) as string[]).some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
-      } catch { return false }
-    })()
-
-    if (hasIndex) {
-      try {
-        const { SemanticSearch } = await import('@soederpop/luca/agi')
-        if (!container.features.available.includes('semanticSearch')) {
-          SemanticSearch.attach(container)
-        }
-        const dbPath = path.join(collection.rootPath, '.contentbase/search.sqlite')
-        const ss = container.feature('semanticSearch', { dbPath }) as any
-        await ss.initDb()
-
-        // Collect document inputs
-        const docs: any[] = []
-        for (const pathId of collection.available) {
-          const doc = collection.document(pathId) as any
-          docs.push({
-            pathId,
-            model: collection.findModelDefinition(pathId)?.name ?? undefined,
-            title: doc.title,
-            meta: doc.meta,
-            content: doc.content,
-          })
-        }
-
-        const stale = docs.filter((d: any) => ss.needsReindex(d))
-        ss.removeStale(docs.map((d: any) => d.pathId))
+        const ss = await getInitializedSemanticSearch(container, collection.rootPath)
+        const docs = collectDocumentInputs(collection)
+        const stale = docs.filter((d) => ss.needsReindex(d))
+        ss.removeStale(docs.map((d) => d.pathId))
 
         if (stale.length > 0) {
           console.log(`[search] Updating ${stale.length} stale document(s)...`)

package/src/cli/index.ts

@@ -12,7 +12,7 @@ async function renderMarkdown(container: any, text: string) {
 
 async function main() {
   // Dynamic import so the library stays luca-free; only the CLI pulls it in
-  const luca = await import('@soederpop/luca/node')
+  const luca = await import('luca/node')
   const container = luca.default
 
   const commandName = container.argv._[0] as string | undefined

package/src/cli/load-collection.ts

@@ -138,10 +138,10 @@ export async function loadCollection(options: {
   const cwd = process.cwd();
 
   // If no container was passed, try to grab the luca singleton.
-  // This works when running inside the cnotes CLI (which imports @soederpop/luca/node).
+  // This works when running inside the cnotes CLI (which imports luca/node).
   if (!container) {
     try {
-      const luca = await import('@soederpop/luca/node');
+      const luca = await import('luca/node');
       container = luca.default;
     } catch {
       // Not running in a luca context — that's fine, native imports will be used

package/src/collection.ts

@@ -1,13 +1,12 @@
-import fs from "fs/promises";
 import path from "path";
 import matter from "gray-matter";
 import picomatch from "picomatch";
 import { Document } from "./document";
 import { CollectionQuery } from "./query/collection-query";
 import { createModelInstance } from "./model-instance";
-import { readDirectory } from "./utils/read-directory";
 import { pluralize } from "./utils/inflect";
 import { Base } from "./base-model";
+import { NodeStorageAdapter } from "./adapters/node-fs";
 import type {
   ModelDefinition,
   CollectionItem,
@@ -15,6 +14,7 @@ import type {
   InferModelInstance,
   HasManyDefinition,
   RelationshipDefinition,
+  StorageAdapter,
 } from "./types";
 
 // ─── Zod schema introspection ───
@@ -110,6 +110,7 @@ export class Collection {
   readonly name: string;
   readonly extensions: string[];
 
+  #adapter: StorageAdapter;
   #items: Map<string, CollectionItem> = new Map();
   #documents: Map<string, Document> = new Map();
   #models: Map<string, ModelDefinition<any, any, any, any, any>> = new Map();
@@ -120,7 +121,13 @@ export class Collection {
   #moduleLoader?: (filePath: string) => Record<string, any> | Promise<Record<string, any>>;
 
   constructor(options: CollectionOptions) {
-    this.rootPath = path.resolve(options.rootPath);
+    this.#adapter = options.adapter ?? new NodeStorageAdapter();
+    // Resolve to an absolute path for local FS (NodeStorageAdapter default).
+    // When a custom adapter is provided, rootPath is the backend-specific root
+    // (e.g. a bucket prefix for R2) and is stored as-is.
+    this.rootPath = options.adapter
+      ? options.rootPath
+      : path.resolve(options.rootPath);
     this.name = options.name ?? options.rootPath;
     this.extensions = options.extensions ?? ["mdx", "md"];
     this.#autoDiscover = options.autoDiscover ?? true;
@@ -191,9 +198,17 @@ export class Collection {
   }
 
   get available(): string[] {
-    return Array.from(this.#items.keys()).filter(
-      (id) => !this.#isExcludedByModel(id)
-    );
+    return Array.from(this.#items.keys())
+      .filter((id) => !this.#isExcludedByModel(id))
+      .sort();
+  }
+
+  /**
+   * Match available pathIds against one or more glob patterns.
+   */
+  matchPaths(patterns: string | string[]): string[] {
+    const isMatch = picomatch(patterns);
+    return this.available.filter((id) => isMatch(id));
   }
 
   /**
@@ -244,27 +259,26 @@ export class Collection {
       `\\.(${this.extensions.join("|")})$`,
       "i"
     );
-    const paths = await readDirectory(this.rootPath, extensionPattern);
+    const entries = await this.#adapter.listFiles(this.rootPath, extensionPattern);
 
     await Promise.all(
-      paths.map(async (filePath) => {
-        const pathId = this.getPathId(filePath);
+      entries.map(async ({ key, stat }) => {
+        const pathId = this.getPathId(key);
 
         // Globally exclude templates directory — templates are only used
         // for scaffolding and should never appear in queries or listings
         if (pathId.startsWith("templates/") || pathId === "templates") return;
 
-        const raw = await fs.readFile(filePath, "utf8");
-        const stat = await fs.stat(filePath);
+        const raw = await this.#adapter.readFile(key);
         const { data, content } = matter(raw);
 
         this.#items.set(pathId, {
           raw,
           content,
           meta: data,
-          path: filePath,
-          createdAt: stat.ctime,
-          updatedAt: stat.mtime,
+          path: key,
+          createdAt: stat.createdAt,
+          updatedAt: stat.updatedAt,
           size: stat.size,
         });
       })
@@ -440,8 +454,7 @@ export class Collection {
     const item = this.#items.get(pathId)!;
     const filePath = item.path;
 
-    await fs.mkdir(path.parse(filePath).dir, { recursive: true });
-    await fs.writeFile(filePath, options.content, "utf8");
+    await this.#adapter.writeFile(filePath, options.content);
 
     // Update the stored item
     item.raw = options.content;
@@ -460,11 +473,7 @@ export class Collection {
     const item = this.#items.get(pathId);
     if (!item) return this;
 
-    try {
-      await fs.rm(item.path);
-    } catch {
-      // File might not exist
-    }
+    await this.#adapter.deleteFile(item.path);
     this.#items.delete(pathId);
     this.#documents.delete(pathId);
     return this;
@@ -482,8 +491,8 @@ export class Collection {
       filePath = this.resolve(`${pathId}.${extension}`);
     }
 
-    const raw = await fs.readFile(filePath, "utf8");
-    const stat = await fs.stat(filePath);
+    const raw = await this.#adapter.readFile(filePath);
+    const stat = await this.#adapter.stat(filePath);
     const { data, content } = matter(raw);
 
     const item: CollectionItem = {
@@ -491,8 +500,8 @@ export class Collection {
       content,
       meta: data,
       path: filePath,
-      createdAt: stat.ctime,
-      updatedAt: stat.mtime,
+      createdAt: stat.createdAt,
+      updatedAt: stat.updatedAt,
       size: stat.size,
     };
 
@@ -542,11 +551,11 @@ export class Collection {
   // ─── Utilities ───
 
   resolve(...args: string[]): string {
-    return path.resolve(this.rootPath, ...args);
+    return this.#adapter.join(this.rootPath, ...args);
   }
 
-  getPathId(absolutePath: string): string {
-    const relativePath = path.relative(this.rootPath, absolutePath);
+  getPathId(key: string): string {
+    const relativePath = this.#adapter.relative(this.rootPath, key);
     return relativePath.replace(/\.[a-z]+$/i, "");
   }
 
@@ -669,7 +678,7 @@ export class Collection {
 
     for (const pathId of sorted) {
       const item = this.#items.get(pathId)!;
-      const ext = path.extname(item.path);
+      const ext = this.#adapter.extname(item.path);
       const fullPath = `${pathId}${ext}`;
       const parts = fullPath.split("/");
 
@@ -703,7 +712,7 @@ export class Collection {
 
   #tocEntry(pathId: string, basePath: string): string {
     const item = this.#items.get(pathId)!;
-    const ext = path.extname(item.path);
+    const ext = this.#adapter.extname(item.path);
     const relativePath = `${basePath}/${pathId}${ext}`;
     const doc = this.document(pathId);
     return `- [${doc.title}](${relativePath})`;
@@ -774,12 +783,12 @@ export class Collection {
    */
   async saveModelSummary(options: { includeIds?: boolean } = {}): Promise<string> {
     const summary = this.generateModelSummary(options);
-    const modelsPath = path.join(this.rootPath, "README.md");
+    const modelsPath = this.#adapter.join(this.rootPath, "README.md");
 
     // Preserve existing Overview section content
     let overview = "";
     try {
-      const existing = await fs.readFile(modelsPath, "utf8");
+      const existing = await this.#adapter.readFile(modelsPath);
       const overviewStart = existing.indexOf("## Overview");
       if (overviewStart !== -1) {
         const contentStart = existing.indexOf("\n", overviewStart) + 1;
@@ -807,7 +816,7 @@ export class Collection {
     ];
 
     const markdown = lines.join("\n");
-    await fs.writeFile(modelsPath, markdown, "utf8");
+    await this.#adapter.writeFile(modelsPath, markdown);
     return summary;
   }
 

package/src/document.ts

@@ -1,6 +1,8 @@
 import { unified } from "unified";
 import remarkParse from "remark-parse";
 import remarkGfm from "remark-gfm";
+import remarkRehype from "remark-rehype";
+import rehypeStringify from "rehype-stringify";
 import yaml from "js-yaml";
 import { toString } from "mdast-util-to-string";
 import { kebabCase } from "./utils/inflect";
@@ -9,6 +11,8 @@ import { NodeShortcuts } from "./node-shortcuts";
 import { stringifyAst } from "./utils/stringify-ast";
 import { normalizeHeadings } from "./utils/normalize-headings";
 import { parseTable } from "./utils/parse-table";
+import { stripMarkdown } from "./utils/strip-markdown";
+import type { StripMarkdownOptions } from "./utils/strip-markdown";
 import type { Root, Content, RootContent, Heading } from "mdast";
 import type { Collection } from "./collection";
 
@@ -443,6 +447,10 @@ export class Document {
     };
   }
 
+  stripMarkdown(options?: StripMarkdownOptions): string {
+    return stripMarkdown(this.ast, options);
+  }
+
   toText(
     filterFn: (node: Content) => boolean = () => true
   ): string {
@@ -457,6 +465,16 @@ export class Document {
    * Each heading is indented based on its depth relative to the
    * minimum heading depth found in the document.
    */
+  async toHtml(): Promise<string> {
+    const result = await unified()
+      .use(remarkParse)
+      .use(remarkGfm)
+      .use(remarkRehype)
+      .use(rehypeStringify)
+      .process(this.content);
+    return String(result);
+  }
+
   toOutline(): string {
     const headings = (this.ast.children as Content[]).filter(
       (n): n is Heading => n.type === "heading"

package/src/extract-sections.ts

@@ -2,6 +2,8 @@ import { toString } from "mdast-util-to-string";
 import { AstQuery } from "./ast-query";
 import { NodeShortcuts } from "./node-shortcuts";
 import { stringifyAst } from "./utils/stringify-ast";
+import { stripMarkdown } from "./utils/strip-markdown";
+import type { StripMarkdownOptions } from "./utils/strip-markdown";
 import type { Root, Content, RootContent, Heading } from "mdast";
 import type { ParsedDocument } from "./parse";
 
@@ -150,6 +152,7 @@ function buildParsedDocument(ast: Root): ParsedDocument {
     nodes,
     title,
     stringify: (tree: Root = ast) => stringifyAst(tree),
+    stripMarkdown: (options?: StripMarkdownOptions) => stripMarkdown(ast, options),
     extractSection,
     querySection,
   };

package/src/index.ts

@@ -39,8 +39,15 @@ export { validateDocument } from "./validator";
 // Pattern matching
 export { matchPattern, matchPatterns } from "./utils/match-pattern";
 
+// Markdown utilities
+export { stripMarkdown } from "./utils/strip-markdown";
+export type { StripMarkdownOptions } from "./utils/strip-markdown";
+
 import { toString } from "mdast-util-to-string";
 
+// Storage adapters
+export { NodeStorageAdapter } from "./adapters/node-fs";
+
 // Types
 export type {
   ModelDefinition,
@@ -57,6 +64,9 @@ export type {
   SerializeOptions,
   SaveOptions,
   DocumentRef,
+  StorageAdapter,
+  FileStat,
+  FileEntry,
 } from "./types";
 
 // Re-export zod for convenience

package/src/mcp/helpers.ts

@@ -0,0 +1,7 @@
+export function errorResult(message: string) {
+  return { content: [{ type: 'text' as const, text: message }], isError: true }
+}
+
+export function textResult(text: string) {
+  return { content: [{ type: 'text' as const, text }] }
+}

package/src/mcp/model-info.ts

@@ -0,0 +1,102 @@
+import { introspectMetaSchema } from '../index.js'
+
+export function generateModelInfo(collection: any, def: any) {
+  const lines: string[] = []
+  const name = def.name as string
+  const prefix = def.prefix as string
+  const description = def.description || ''
+  const prefixDocs = collection.available.filter((id: string) => id.startsWith(prefix + '/'))
+
+  lines.push(`# Model: ${name}`, '')
+  lines.push(`- **Prefix:** \`${prefix}/\``)
+  lines.push(`- **Documents:** ${prefixDocs.length}`)
+  if (description) lines.push(`- **Description:** ${description}`)
+  lines.push('')
+
+  // Fields
+  const fields = introspectMetaSchema(def.meta)
+  if (fields.length > 0) {
+    lines.push('## Frontmatter Fields', '')
+    lines.push('| Field | Type | Required | Default | Description |')
+    lines.push('|-------|------|----------|---------|-------------|')
+    for (const f of fields as any[]) {
+      const req = f.required ? 'yes' : 'no'
+      const def_val = f.defaultValue !== undefined ? `\`${JSON.stringify(f.defaultValue)}\`` : ''
+      const desc = f.description || ''
+      lines.push(`| ${f.name} | ${f.type} | ${req} | ${def_val} | ${desc} |`)
+    }
+    lines.push('')
+  }
+
+  // Sections
+  const sections = Object.entries(def.sections || {})
+  if (sections.length > 0) {
+    lines.push('## Sections', '')
+    for (const [key, sec] of sections as [string, any][]) {
+      lines.push(`- **${sec.heading}** (key: \`${key}\`)${sec.schema ? ' — has schema validation' : ''}`)
+      if (sec.alternatives?.length) {
+        lines.push(`  Alternatives: ${sec.alternatives.join(', ')}`)
+      }
+    }
+    lines.push('')
+  }
+
+  // Relationships
+  const relationships = Object.entries(def.relationships || {})
+  if (relationships.length > 0) {
+    lines.push('## Relationships', '')
+    for (const [key, rel] of relationships as [string, any][]) {
+      lines.push(`- \`${key}\` → ${rel.type} **${rel.model}**`)
+    }
+    lines.push('')
+  }
+
+  // Computed & Scopes
+  const computedKeys = Object.keys(def.computed || {})
+  if (computedKeys.length > 0) {
+    lines.push('## Computed Properties', '')
+    lines.push(computedKeys.map(k => `- \`${k}\``).join('\n'))
+    lines.push('')
+  }
+
+  const scopeKeys = Object.keys(def.scopes || {})
+  if (scopeKeys.length > 0) {
+    lines.push('## Named Scopes', '')
+    lines.push(scopeKeys.map(k => `- \`${k}\``).join('\n'))
+    lines.push('')
+  }
+
+  // Existing documents
+  if (prefixDocs.length > 0) {
+    lines.push('## Existing Documents', '')
+    for (const id of prefixDocs) {
+      lines.push(`- \`${id}\``)
+    }
+    lines.push('')
+  }
+
+  // Example scaffold
+  const defaultMeta: Record<string, any> = {}
+  for (const f of fields as any[]) {
+    if (f.defaultValue !== undefined) {
+      defaultMeta[f.name] = f.defaultValue
+    } else if (f.required) {
+      defaultMeta[f.name] = `<${f.type}>`
+    }
+  }
+  const sectionHeadings = sections.map(([, sec]: [string, any]) => `## ${sec.heading}\n\n`)
+
+  lines.push('## Example Document', '')
+  lines.push('```markdown')
+  lines.push('---')
+  for (const [k, v] of Object.entries(defaultMeta)) {
+    lines.push(`${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`)
+  }
+  lines.push('---')
+  lines.push(`# Your Title Here`)
+  lines.push('')
+  lines.push(sectionHeadings.join(''))
+  lines.push('```')
+
+  return lines.join('\n')
+}