contentbase 0.0.2 → 0.0.4

package/src/api/endpoints/semantic-search.ts

@@ -0,0 +1,120 @@
+import { z } from 'zod'
+import pathModule from 'node:path'
+import { existsSync, readdirSync } from 'node:fs'
+
+export const path = '/api/semantic-search'
+export const description = 'Semantic search across collection documents'
+export const tags = ['query']
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function hasSearchIndex(rootPath: string): boolean {
+  const dbDir = pathModule.join(rootPath, '.contentbase')
+  if (!existsSync(dbDir)) return false
+  try {
+    const files = readdirSync(dbDir) as string[]
+    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
+  } catch {
+    return false
+  }
+}
+
+let _semanticSearch: any = null
+
+async function getSemanticSearch(container: any, rootPath: string) {
+  if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
+
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  if (!container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(container)
+  }
+
+  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
+  _semanticSearch = container.feature('semanticSearch', { dbPath })
+  await _semanticSearch.initDb()
+  return _semanticSearch
+}
+
+// ── GET /api/semantic-search ─────────────────────────────────────────
+
+export const getSchema = z.object({
+  q: z.string(),
+  mode: z.string().optional(),
+  model: z.string().optional(),
+  limit: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const limit = params.limit ? parseInt(params.limit, 10) : 10
+  const searchOptions = { limit, model: params.model }
+
+  let results: any[]
+  switch (mode) {
+    case 'keyword':
+      results = await ss.search(params.q, searchOptions)
+      break
+    case 'vector':
+      results = await ss.vectorSearch(params.q, searchOptions)
+      break
+    case 'hybrid':
+    default:
+      results = await ss.hybridSearch(params.q, searchOptions)
+      break
+  }
+
+  return results
+}
+
+// ── POST /api/semantic-search ────────────────────────────────────────
+
+export const postSchema = z.object({
+  query: z.string(),
+  mode: z.enum(['hybrid', 'keyword', 'vector']).optional(),
+  model: z.string().optional(),
+  limit: z.number().optional(),
+  where: z.record(z.string(), z.any()).optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const searchOptions = {
+    limit: params.limit || 10,
+    model: params.model,
+    where: params.where,
+  }
+
+  let results: any[]
+  switch (mode) {
+    case 'keyword':
+      results = await ss.search(params.query, searchOptions)
+      break
+    case 'vector':
+      results = await ss.vectorSearch(params.query, searchOptions)
+      break
+    case 'hybrid':
+    default:
+      results = await ss.hybridSearch(params.query, searchOptions)
+      break
+  }
+
+  return results
+}

package/src/api/endpoints/text-search.ts

@@ -0,0 +1,63 @@
+import { z } from 'zod'
+
+export const path = '/api/text-search'
+export const description = 'Search file contents with pattern matching'
+export const tags = ['query']
+
+export const getSchema = z.object({
+  pattern: z.string(),
+  expanded: z.string().optional(),
+  include: z.string().optional(),
+  exclude: z.string().optional(),
+  ignoreCase: z.string().optional(),
+  maxResults: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const grep = ctx.container.feature('grep')
+  const searchPath = collection.rootPath
+
+  const expanded = params.expanded === 'true'
+  const ignoreCase = params.ignoreCase === 'true'
+  const maxResults = params.maxResults ? parseInt(params.maxResults, 10) : undefined
+
+  if (!expanded) {
+    const files = await grep.filesContaining(params.pattern, {
+      path: searchPath,
+      ignoreCase,
+      maxResults,
+      include: params.include,
+      exclude: params.exclude,
+    })
+
+    return { files, count: files.length }
+  }
+
+  const results = await grep.search({
+    pattern: params.pattern,
+    path: searchPath,
+    ignoreCase,
+    maxResults,
+    include: params.include,
+    exclude: params.exclude,
+  })
+
+  // Group by file
+  const grouped = new Map<string, Array<{ line: number; column?: number; content: string }>>()
+  for (const match of results) {
+    if (!grouped.has(match.file)) grouped.set(match.file, [])
+    grouped.get(match.file)!.push({
+      line: match.line,
+      column: match.column,
+      content: match.content,
+    })
+  }
+
+  const files = Array.from(grouped.entries()).map(([file, matches]) => ({
+    file,
+    matches,
+  }))
+
+  return { files, count: files.length }
+}

package/src/api/endpoints/validate.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod'
+import { validateDocument } from '../../index.js'
+import { resolveModelDef } from '../helpers.js'
+
+export const path = '/api/validate'
+export const description = 'Validate a document against its model schema'
+export const tags = ['validation']
+
+export const getSchema = z.object({
+  pathId: z.string(),
+  model: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+
+  if (!collection.available.includes(params.pathId)) {
+    ctx.response.status(404)
+    return { error: `Document not found: ${params.pathId}` }
+  }
+
+  const doc = collection.document(params.pathId)
+
+  const def = params.model
+    ? resolveModelDef(collection, params.model)
+    : collection.findModelDefinition(params.pathId)
+
+  if (!def) {
+    ctx.response.status(400)
+    return { error: `No model definition found for ${params.pathId}` }
+  }
+
+  return validateDocument(doc, def)
+}

package/src/api/helpers.ts

@@ -0,0 +1,97 @@
+import { unified } from 'unified'
+import remarkParse from 'remark-parse'
+import remarkGfm from 'remark-gfm'
+import remarkRehype from 'remark-rehype'
+import rehypeStringify from 'rehype-stringify'
+import type { Collection } from '../collection.js'
+import { introspectMetaSchema } from '../collection.js'
+
+export async function renderMarkdownToHtml(markdown: string): Promise<string> {
+  const result = await unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkRehype)
+    .use(rehypeStringify)
+    .process(markdown)
+  return String(result)
+}
+
+export function rewriteDocLinks(html: string): string {
+  return html.replace(/href="([^"]*\.(?:md|mdx))"/g, (match, href) => {
+    if (href.startsWith('http://') || href.startsWith('https://') || href.startsWith('//')) {
+      return match
+    }
+    return `href="${href.replace(/\.mdx?$/, '')}"`
+  })
+}
+
+export const tocPageStyle = `<style>
+  *, *::before, *::after { box-sizing: border-box; }
+  body {
+    font-family: 'Inter', system-ui, -apple-system, sans-serif;
+    max-width: 52rem;
+    margin: 0 auto;
+    padding: 2rem 1.5rem;
+    line-height: 1.7;
+    color: #1a1a2e;
+    background: #fafafa;
+  }
+  h1 { font-size: 2rem; font-weight: 600; margin: 2rem 0 1rem; color: #0f0f23; }
+  h2 { font-size: 1.5rem; font-weight: 600; margin: 2.5rem 0 0.75rem; color: #16163a; border-bottom: 1px solid #e2e2e8; padding-bottom: 0.4rem; }
+  a { color: #2563eb; text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  ul { padding-left: 1.5rem; margin: 0.5rem 0 1rem; }
+  li { margin: 0.3rem 0; }
+</style>`
+
+export async function renderTocPage(collection: Collection): Promise<string> {
+  const tocMarkdown = collection.tableOfContents({ title: 'Table of Contents', basePath: '.' })
+  const tocHtml = rewriteDocLinks(await renderMarkdownToHtml(tocMarkdown))
+  return `<!DOCTYPE html>
+<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Table of Contents</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&display=swap" rel="stylesheet">
+${tocPageStyle}
+</head>
+<body>
+${tocHtml}
+</body></html>`
+}
+
+export function resolveModelDef(collection: Collection, name: string) {
+  const lower = name.toLowerCase()
+  return collection.modelDefinitions.find(
+    (d: any) => d.name.toLowerCase() === lower || d.prefix.toLowerCase() === lower,
+  )
+}
+
+export function buildSchemaJSON(collection: Collection) {
+  const models: Record<string, any> = {}
+  for (const def of collection.modelDefinitions as any[]) {
+    const fields = introspectMetaSchema(def.meta)
+    const sections = Object.entries(def.sections || {}).map(([key, sec]: [string, any]) => ({
+      key,
+      heading: sec.heading,
+      alternatives: sec.alternatives || [],
+      hasSchema: !!sec.schema,
+    }))
+    const relationships = Object.entries(def.relationships || {}).map(([key, rel]: [string, any]) => ({
+      key,
+      type: rel.type,
+      model: rel.model,
+    }))
+    models[def.name] = {
+      name: def.name,
+      prefix: def.prefix,
+      description: def.description,
+      fields,
+      sections,
+      relationships,
+      computed: Object.keys(def.computed || {}),
+      scopes: Object.keys(def.scopes || {}),
+    }
+  }
+  return models
+}

package/src/base-model.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+import { defineModel } from "./define-model";
+
+/**
+ * Built-in Base model that serves as a catch-all for documents
+ * that don't match any other registered model.
+ * Auto-registered by Collection.load() unless overridden.
+ */
+export const Base = defineModel("Base", {
+  prefix: "",
+  meta: z.looseObject({}),
+});

package/src/cli/commands/action.ts

@@ -1,44 +1,82 @@
-import { defineCommand } from "citty";
-import { loadCollection } from "../load-collection";
-
-export default defineCommand({
-  meta: {
-    name: "action",
-    description: "Run a named action on the collection",
-  },
-  args: {
-    name: {
-      type: "positional",
-      description: "Action name",
-      required: true,
-    },
-    rootPath: {
-      type: "string",
-      description: "Root path for the collection",
-      alias: "r",
-    },
-  },
-  async run({ args }) {
-    const collection = await loadCollection({
-      rootPath: args.rootPath as string | undefined,
-    });
-
-    const actionName = args.name as string;
-
-    if (!collection.actions.has(actionName)) {
-      console.error(
-        `Action "${actionName}" not found. Available: ${collection.availableActions.join(", ") || "(none)"}`
-      );
-      process.exit(1);
-    }
-
-    const result = await collection.runAction(actionName);
-    if (result !== undefined) {
-      console.log(
-        typeof result === "string"
-          ? result
-          : JSON.stringify(result, null, 2)
-      );
-    }
-  },
-});
+import { z } from 'zod'
+import { commands } from '../registry.js'
+import { loadCollection } from '../load-collection.js'
+
+const argsSchema = z.object({
+  contentFolder: z.string().optional(),
+})
+
+async function handler(options: z.infer<typeof argsSchema>, context: { container: any }) {
+  const collection = await loadCollection({
+    contentFolder: options.contentFolder,
+  })
+
+  const actionName = context.container.argv._[1] as string | undefined
+
+  if (!actionName) {
+    console.error('Usage: cnotes action <name>')
+    console.error(`Available: ${collection.availableActions.join(', ') || '(none)'}`)
+    process.exit(1)
+  }
+
+  if (!collection.actions.has(actionName)) {
+    console.error(
+      `Action "${actionName}" not found. Available: ${collection.availableActions.join(', ') || '(none)'}`
+    )
+    process.exit(1)
+  }
+
+  const result = await collection.runAction(actionName)
+  if (result !== undefined) {
+    console.log(
+      typeof result === 'string'
+        ? result
+        : JSON.stringify(result, null, 2)
+    )
+  }
+}
+
+commands.register('action', {
+  description: 'Run a named action on the collection',
+  help: `# cnotes action
+
+Run a named action registered on the collection. Actions are custom functions defined in your collection's entry point.
+
+## Usage
+
+\`\`\`
+cnotes action <name> [options]
+\`\`\`
+
+## Arguments
+
+| Argument | Description |
+|----------|-------------|
+| \`name\` | The registered action name |
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| \`--contentFolder\` | Path to content folder |
+
+## Output
+
+If the action returns a value, it is printed to stdout (strings directly, objects as JSON).
+
+## Examples
+
+\`\`\`bash
+# Run an action
+cnotes action generate-report
+
+# Run an action on a specific content folder
+cnotes action sync --contentFolder ./docs
+
+# List available actions (shows error with available names)
+cnotes action
+\`\`\`
+`,
+  argsSchema,
+  handler,
+})

package/src/cli/commands/console.ts

@@ -0,0 +1,124 @@
+import { z } from 'zod'
+import { commands } from '../registry.js'
+import { loadCollection } from '../load-collection.js'
+
+const argsSchema = z.object({
+  contentFolder: z.string().optional(),
+})
+
+async function handler(options: z.infer<typeof argsSchema>, context: { container: any }) {
+  const container = context.container
+  const ui = container.feature('ui')
+
+  const collection = await loadCollection({
+    contentFolder: options.contentFolder,
+  })
+
+  const featureContext: Record<string, any> = {
+    collection,
+  }
+
+  // Expose all container features
+  for (const name of container.features.available) {
+    try {
+      featureContext[name] = container.feature(name)
+    } catch {}
+  }
+
+  // Load user console module if present
+  const consoleModulePath = container.paths.resolve('cnotes.console.ts')
+  let consoleModuleLoaded = false
+  let consoleModuleError: Error | null = null
+
+  if (container.fs.exists(consoleModulePath)) {
+    try {
+      const vmFeature = container.feature('vm')
+      const userExports = vmFeature.loadModule(consoleModulePath, { container, console, collection })
+      Object.assign(featureContext, userExports)
+      consoleModuleLoaded = true
+    } catch (err: any) {
+      consoleModuleError = err
+    }
+  }
+
+  const prompt = ui.colors.cyan('cnotes') + ui.colors.dim(' > ')
+
+  console.log()
+  console.log(ui.colors.dim('  Contentbase REPL — collection and container features in scope. Tab to autocomplete.'))
+  if (consoleModuleLoaded) {
+    console.log(ui.colors.dim('  Loaded cnotes.console.ts exports into scope.'))
+  } else if (consoleModuleError) {
+    console.log(ui.colors.yellow('  Warning: Failed to load cnotes.console.ts:'))
+    console.log(ui.colors.yellow(`    ${consoleModuleError.message}`))
+    console.log(ui.colors.dim('  The REPL will start without your custom exports.'))
+  }
+  console.log(ui.colors.dim('  Type .exit to quit.'))
+  console.log()
+
+  const repl = container.feature('repl', { prompt })
+  await repl.start({
+    context: {
+      ...featureContext,
+      console,
+      setTimeout,
+      setInterval,
+      clearTimeout,
+      clearInterval,
+      fetch,
+      Bun,
+    },
+  })
+}
+
+commands.register('console', {
+  description: 'Start an interactive REPL with collection and container features in scope',
+  help: `# cnotes console
+
+Start an interactive REPL with the loaded collection, all container features, and optional user-defined exports in scope. Useful for exploring and debugging your content.
+
+## Usage
+
+\`\`\`
+cnotes console [options]
+\`\`\`
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| \`--contentFolder\` | Path to content folder |
+
+## Scope
+
+The REPL has these variables available:
+
+- \`collection\` — The loaded Collection instance
+- All luca container features (fs, ui, grep, etc.)
+- Exports from \`cnotes.console.ts\` if present in the project root
+
+## Custom Console Module
+
+Create a \`cnotes.console.ts\` file in your project root to add custom helpers to the REPL scope:
+
+\`\`\`typescript
+export const myHelper = () => "hello from console"
+\`\`\`
+
+## Examples
+
+\`\`\`bash
+# Start the REPL
+cnotes console
+
+# Start with a specific content folder
+cnotes console --contentFolder ./docs
+
+# Once inside the REPL:
+#   collection.available           — list all document IDs
+#   collection.document('posts/hello')  — load a document
+#   .exit                          — quit
+\`\`\`
+`,
+  argsSchema,
+  handler,
+})