contentbase 0.0.1 → 0.0.4

package/src/cli/commands/embed.ts

@@ -0,0 +1,323 @@
+import { z } from 'zod'
+import { existsSync, readdirSync, accessSync, constants as fsConstants } from 'node:fs'
+import path from 'node:path'
+import { commands } from '../registry.js'
+import { loadCollection } from '../load-collection.js'
+
+const argsSchema = z.object({
+  force: z.boolean().default(false),
+  provider: z.enum(['local', 'openai']).default('openai'),
+  status: z.boolean().default(false),
+  local: z.boolean().default(false),
+  installLocal: z.boolean().default(false),
+  contentFolder: z.string().optional(),
+})
+
+function hasSearchIndex(rootPath: string): boolean {
+  const dbDir = path.join(rootPath, '.contentbase')
+  if (!existsSync(dbDir)) return false
+  try {
+    const files = readdirSync(dbDir) as string[]
+    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
+  } catch {
+    return false
+  }
+}
+
+function collectDocumentInputs(collection: any) {
+  const inputs: any[] = []
+  for (const pathId of collection.available) {
+    const doc = collection.document(pathId)
+    const modelDef = collection.findModelDefinition(pathId)
+
+    const sections: any[] = []
+    const lines = (doc.content as string).split('\n')
+    let currentHeading: string | null = null
+    let currentContent: string[] = []
+
+    for (const line of lines) {
+      const h2Match = line.match(/^## (.+)/)
+      if (h2Match) {
+        if (currentHeading) {
+          sections.push({
+            heading: currentHeading,
+            headingPath: currentHeading,
+            content: currentContent.join('\n').trim(),
+            level: 2,
+          })
+        }
+        currentHeading = h2Match[1].trim()
+        currentContent = []
+      } else if (currentHeading) {
+        currentContent.push(line)
+      }
+    }
+    if (currentHeading) {
+      sections.push({
+        heading: currentHeading,
+        headingPath: currentHeading,
+        content: currentContent.join('\n').trim(),
+        level: 2,
+      })
+    }
+
+    inputs.push({
+      pathId,
+      model: modelDef?.name ?? undefined,
+      title: doc.title,
+      slug: doc.slug,
+      meta: doc.meta,
+      content: doc.content,
+      sections: sections.length > 0 ? sections : undefined,
+    })
+  }
+  return inputs
+}
+
+function isLocalInstalled(): boolean {
+  const modulePath = path.join(process.cwd(), 'node_modules', 'node-llama-cpp')
+  try {
+    return existsSync(modulePath)
+  } catch {
+    return false
+  }
+}
+
+async function installLocal(SemanticSearch: any, container: any): Promise<void> {
+  const cwd = process.cwd()
+  const nodeModules = path.join(cwd, 'node_modules')
+
+  // Check permissions
+  try {
+    if (existsSync(nodeModules)) {
+      accessSync(nodeModules, fsConstants.W_OK)
+    }
+  } catch {
+    const version = SemanticSearch.PINNED_LLAMA_VERSION
+    const cmd = detectInstallCommand(cwd, version)
+    console.error(`Cannot write to ${nodeModules}.`)
+    console.error(`Run manually with elevated permissions:\n  ${cmd}`)
+    process.exit(1)
+  }
+
+  console.error(`Installing node-llama-cpp@${SemanticSearch.PINNED_LLAMA_VERSION}...`)
+
+  // Create a temporary instance to use the install method
+  if (!container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(container)
+  }
+  const ss = container.feature('semanticSearch', {
+    dbPath: path.join(cwd, '.contentbase/search.sqlite'),
+    embeddingProvider: 'local',
+  })
+
+  try {
+    await ss.installLocalEmbeddings(cwd)
+    console.log(`node-llama-cpp@${SemanticSearch.PINNED_LLAMA_VERSION} installed successfully.`)
+  } catch (err: any) {
+    const msg = err?.message ?? String(err)
+    if (msg.includes('network') || msg.includes('ETIMEDOUT') || msg.includes('ENOTFOUND')) {
+      console.error(`Install failed (network unreachable). To use local embeddings offline, manually install node-llama-cpp@${SemanticSearch.PINNED_LLAMA_VERSION} into this project.`)
+    } else {
+      console.error(msg)
+    }
+    process.exit(1)
+  }
+}
+
+function detectInstallCommand(cwd: string, version: string): string {
+  const pkg = `node-llama-cpp@${version}`
+  if (existsSync(path.join(cwd, 'bun.lockb')) || existsSync(path.join(cwd, 'bun.lock'))) {
+    return `bun add --optional ${pkg}`
+  } else if (existsSync(path.join(cwd, 'pnpm-lock.yaml'))) {
+    return `pnpm add --save-optional ${pkg}`
+  } else if (existsSync(path.join(cwd, 'yarn.lock'))) {
+    return `yarn add --optional ${pkg}`
+  }
+  return `npm install --save-optional ${pkg}`
+}
+
+async function handler(options: z.infer<typeof argsSchema>, { container }: { container: any }) {
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+
+  // --install-local: install node-llama-cpp only, then exit
+  if (options.installLocal) {
+    if (isLocalInstalled()) {
+      console.log('node-llama-cpp is already installed. Skipping.')
+      return
+    }
+    await installLocal(SemanticSearch, container)
+    return
+  }
+
+  // Resolve effective provider: --local flag overrides --provider
+  const provider = options.local ? 'local' : options.provider
+
+  // --local: auto-install if needed
+  if (options.local && !isLocalInstalled()) {
+    await installLocal(SemanticSearch, container)
+  }
+
+  const collection = await loadCollection({
+    contentFolder: options.contentFolder,
+  })
+
+  // --status: show index health without modifying anything
+  if (options.status) {
+    if (!hasSearchIndex(collection.rootPath)) {
+      console.log('No search index found.')
+      console.log(`Documents in collection: ${collection.available.length}`)
+      console.log(`\nRun: cnotes embed`)
+      return
+    }
+
+    if (!container.features.available.includes('semanticSearch')) {
+      SemanticSearch.attach(container)
+    }
+    const dbPath = path.join(collection.rootPath, '.contentbase/search.sqlite')
+    const ss = container.feature('semanticSearch', {
+      dbPath,
+      embeddingProvider: provider,
+    })
+    await ss.initDb()
+
+    const stats = ss.getStats()
+    console.log('Search Index Status')
+    console.log('-------------------')
+    console.log(`Documents indexed:  ${stats.documentCount}`)
+    console.log(`Chunks created:     ${stats.chunkCount}`)
+    console.log(`Embeddings stored:  ${stats.embeddingCount}`)
+    console.log(`Last indexed:       ${stats.lastIndexedAt ?? 'never'}`)
+    console.log(`Provider:           ${stats.provider}`)
+    console.log(`Model:              ${stats.model}`)
+    console.log(`Dimensions:         ${stats.dimensions}`)
+    console.log(`Database size:      ${formatBytes(stats.dbSizeBytes)}`)
+    console.log(`Collection docs:    ${collection.available.length}`)
+
+    // Check for staleness
+    const docs = collectDocumentInputs(collection)
+    const stale = docs.filter((d: any) => ss.needsReindex(d)).length
+    if (stale > 0) {
+      console.log(`Stale documents:    ${stale}`)
+    } else {
+      console.log(`Status:             up to date`)
+    }
+    return
+  }
+
+  // Embed / re-embed
+  if (!container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(container)
+  }
+  const dbPath = path.join(collection.rootPath, '.contentbase/search.sqlite')
+  const ss = container.feature('semanticSearch', {
+    dbPath,
+    embeddingProvider: provider,
+  })
+  await ss.initDb()
+
+  const docs = collectDocumentInputs(collection)
+  const startTime = Date.now()
+
+  if (options.force) {
+    console.error('Clearing existing index...')
+    await ss.reindex()
+  }
+
+  const toIndex = options.force ? docs : docs.filter((d: any) => ss.needsReindex(d))
+
+  // Remove stale
+  ss.removeStale(docs.map((d: any) => d.pathId))
+
+  if (toIndex.length === 0) {
+    console.log('All documents are up to date. Nothing to embed.')
+    const stats = ss.getStats()
+    printStats(stats, 0, Date.now() - startTime)
+    return
+  }
+
+  console.error(`Embedding ${toIndex.length} of ${docs.length} document(s)...`)
+  console.error(`Provider: ${provider}`)
+
+  const batchSize = 5
+
+  for (let i = 0; i < toIndex.length; i += batchSize) {
+    const batch = toIndex.slice(i, i + batchSize)
+    await ss.indexDocuments(batch)
+    const done = Math.min(i + batchSize, toIndex.length)
+    const pct = Math.round((done / toIndex.length) * 100)
+    const elapsed = Date.now() - startTime
+    const eta = toIndex.length > done
+      ? Math.round((elapsed / done) * (toIndex.length - done) / 1000)
+      : 0
+    const etaStr = eta > 0 ? ` ETA: ${eta}s` : ''
+    console.error(`  [${pct}%] ${done}/${toIndex.length} documents${etaStr}`)
+  }
+
+  const elapsed = Date.now() - startTime
+  const stats = ss.getStats()
+  printStats(stats, toIndex.length, elapsed)
+}
+
+function printStats(stats: any, indexed: number, elapsedMs: number) {
+  console.log()
+  console.log('Embedding complete.')
+  if (indexed > 0) console.log(`  Documents indexed: ${indexed}`)
+  console.log(`  Total documents:   ${stats.documentCount}`)
+  console.log(`  Total chunks:      ${stats.chunkCount}`)
+  console.log(`  Embeddings:        ${stats.embeddingCount}`)
+  console.log(`  Time elapsed:      ${(elapsedMs / 1000).toFixed(1)}s`)
+  console.log(`  Database size:     ${formatBytes(stats.dbSizeBytes)}`)
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
+}
+
+commands.register('embed', {
+  description: 'Generate embeddings for collection documents',
+  help: `# cnotes embed
+
+Generate or update vector embeddings for all documents in the collection. Required before using \`cnotes search\`.
+
+## Usage
+
+\`\`\`
+cnotes embed [options]
+\`\`\`
+
+## Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| \`--force\` | \`false\` | Re-embed everything (ignore content hashes) |
+| \`--provider\` | \`openai\` | Embedding provider: \`openai\` or \`local\` |
+| \`--status\` | \`false\` | Show index health without embedding |
+| \`--local\` | \`false\` | Use local embeddings; auto-installs node-llama-cpp if not found |
+| \`--install-local\` | \`false\` | Only install node-llama-cpp (no embedding), then exit |
+| \`--contentFolder\` | | Path to content folder |
+
+## Examples
+
+\`\`\`bash
+# Generate/update embeddings (uses OpenAI by default)
+cnotes embed
+
+# Re-embed everything from scratch
+cnotes embed --force
+
+# Use local embeddings (auto-installs node-llama-cpp if needed)
+cnotes embed --local
+
+# Only install node-llama-cpp without embedding
+cnotes embed --install-local
+
+# Show index health
+cnotes embed --status
+\`\`\`
+`,
+  argsSchema,
+  handler,
+})

package/src/cli/commands/export.ts

@@ -1,24 +1,58 @@
-import { defineCommand } from "citty";
-import { loadCollection } from "../load-collection";
-
-export default defineCommand({
-  meta: {
-    name: "export",
-    description: "Export collection as JSON",
-  },
-  args: {
-    rootPath: {
-      type: "string",
-      description: "Root path for the collection",
-      alias: "r",
-    },
-  },
-  async run({ args }) {
-    const collection = await loadCollection({
-      rootPath: args.rootPath as string | undefined,
-    });
-
-    const data = await collection.export();
-    console.log(JSON.stringify(data, null, 2));
-  },
-});
+import { z } from 'zod'
+import { commands } from '../registry.js'
+import { loadCollection } from '../load-collection.js'
+
+const argsSchema = z.object({
+  contentFolder: z.string().optional(),
+})
+
+async function handler(options: z.infer<typeof argsSchema>) {
+  const collection = await loadCollection({
+    contentFolder: options.contentFolder,
+  })
+
+  const data = await collection.export()
+  console.log(JSON.stringify(data, null, 2))
+}
+
+commands.register('export', {
+  description: 'Export collection as JSON',
+  help: `# cnotes export
+
+Export the entire collection as a JSON object. Each document includes its path ID, title, frontmatter, content, and model info.
+
+## Usage
+
+\`\`\`
+cnotes export [options]
+\`\`\`
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| \`--contentFolder\` | Path to content folder |
+
+## Output
+
+Writes JSON to stdout. Pipe to a file or another tool.
+
+## Examples
+
+\`\`\`bash
+# Export to stdout
+cnotes export
+
+# Save to a file
+cnotes export > backup.json
+
+# Export a specific content folder
+cnotes export --contentFolder ./docs > docs.json
+
+# Pipe to jq for filtering
+cnotes export | jq '.[] | select(.meta.status == "published")'
+\`\`\`
+`,
+  argsSchema,
+  handler,
+})

package/src/cli/commands/extract.ts

@@ -0,0 +1,174 @@
+import { z } from 'zod'
+import picomatch from 'picomatch'
+import yaml from 'js-yaml'
+import { commands } from '../registry.js'
+import { loadCollection } from '../load-collection.js'
+import { stringifyAst } from '../../utils/stringify-ast.js'
+import type { Root, RootContent, Heading } from 'mdast'
+
+const argsSchema = z.object({
+  sections: z.string().optional(),
+  s: z.string().optional(),
+  title: z.string().optional(),
+  t: z.string().optional(),
+  frontmatter: z.boolean().default(false),
+  noNormalizeHeadings: z.boolean().default(false),
+  contentFolder: z.string().optional(),
+})
+
+async function handler(options: z.infer<typeof argsSchema>, context: { container: any }) {
+  const collection = await loadCollection({
+    contentFolder: options.contentFolder,
+  })
+
+  const target = context.container.argv._[1] as string | undefined
+  if (!target) {
+    console.error('Usage: cnotes extract <target> --sections "Section1,Section2"')
+    process.exit(1)
+  }
+
+  const sectionsArg = options.sections || options.s
+  if (!sectionsArg) {
+    console.error('--sections is required')
+    process.exit(1)
+  }
+
+  const sectionNames = sectionsArg
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean)
+  const includeFrontmatter = options.frontmatter
+  const noNormalize = options.noNormalizeHeadings
+  const title = options.title || options.t
+
+  const isMatch = picomatch(target)
+  const matchingIds = collection.available.filter((id) => isMatch(id))
+
+  if (matchingIds.length === 0) {
+    console.error(`No documents matched: ${target}`)
+    process.exit(1)
+  }
+
+  const allNodes: RootContent[] = []
+
+  if (title) {
+    allNodes.push({
+      type: 'heading',
+      depth: 1,
+      children: [{ type: 'text', value: title }],
+    } as Heading)
+  }
+
+  for (const id of matchingIds) {
+    const doc = collection.document(id)
+
+    if (includeFrontmatter && Object.keys(doc.meta).length > 0) {
+      allNodes.push({
+        type: 'yaml',
+        value: yaml.dump(doc.meta).trim(),
+      } as any)
+    }
+
+    const docNodes: RootContent[] = []
+
+    const titleNode = doc.nodes.firstHeading
+    if (titleNode) {
+      docNodes.push(titleNode as RootContent)
+    }
+
+    const leading = doc.nodes.leadingElementsAfterTitle
+    if (leading.length > 0) {
+      docNodes.push(...(leading as RootContent[]))
+    }
+
+    for (const name of sectionNames) {
+      try {
+        const sectionNodes = doc.extractSection(name)
+        if (sectionNodes.length > 0) {
+          docNodes.push(...(sectionNodes as RootContent[]))
+        }
+      } catch {
+        // Section not found — skip
+      }
+    }
+
+    if (!noNormalize && docNodes.length > 0) {
+      const headings = docNodes.filter(
+        (n): n is Heading => n.type === 'heading'
+      )
+      if (headings.length > 0) {
+        const minDepth = Math.min(...headings.map((h) => h.depth))
+        const targetDepth = title ? 2 : 1
+        const shift = targetDepth - minDepth
+        if (shift !== 0) {
+          for (const h of headings) {
+            h.depth = Math.max(1, Math.min(6, h.depth + shift)) as
+              | 1
+              | 2
+              | 3
+              | 4
+              | 5
+              | 6
+          }
+        }
+      }
+    }
+
+    allNodes.push(...docNodes)
+  }
+
+  const combinedAst: Root = { type: 'root', children: allNodes }
+  console.log(stringifyAst(combinedAst).trim())
+}
+
+commands.register('extract', {
+  description: 'Extract specific sections from documents',
+  help: `# cnotes extract
+
+Extract specific sections from one or more documents. Supports glob patterns for matching multiple documents and outputs clean markdown with normalized headings.
+
+## Usage
+
+\`\`\`
+cnotes extract <target> --sections "Section1,Section2" [options]
+\`\`\`
+
+## Arguments
+
+| Argument | Description |
+|----------|-------------|
+| \`target\` | Path ID or glob pattern (e.g. \`epics/*\`, \`tasks/auth-*\`) |
+
+## Options
+
+| Option | Alias | Description |
+|--------|-------|-------------|
+| \`--sections\` | \`-s\` | **Required.** Comma-separated section headings to extract |
+| \`--title\` | \`-t\` | Add a top-level H1 title to the output |
+| \`--frontmatter\` | | Include YAML frontmatter in the output |
+| \`--noNormalizeHeadings\` | | Skip heading depth normalization |
+| \`--contentFolder\` | | Path to content folder |
+
+## Heading Normalization
+
+By default, extracted headings are re-leveled so the output starts at H1 (or H2 when \`--title\` is used). Use \`--noNormalizeHeadings\` to preserve original depths.
+
+## Examples
+
+\`\`\`bash
+# Extract Overview from a single document
+cnotes extract epics/auth-system --sections "Overview"
+
+# Extract multiple sections
+cnotes extract tasks/login-bug -s "Overview,Requirements,Acceptance Criteria"
+
+# Extract from all epics with a title wrapper
+cnotes extract "epics/*" -s "Overview" -t "Epic Summaries"
+
+# Include frontmatter in output
+cnotes extract epics/auth-system -s "Overview" --frontmatter
+\`\`\`
+`,
+  argsSchema,
+  handler,
+})

package/src/cli/commands/help.ts

@@ -0,0 +1,81 @@
+import { commands } from '../registry.js'
+
+async function handler(_options: any, context: { container: any }) {
+  const ui = context.container.feature('ui')
+
+  const lines: string[] = [
+    '# cnotes',
+    '',
+    'An ORM for structured Markdown/MDX files.',
+    '',
+    '## Usage',
+    '',
+    '```',
+    'cnotes <command> [options]',
+    '```',
+    '',
+    '## Commands',
+    '',
+    '| Command | Description |',
+    '|---------|-------------|',
+  ]
+
+  for (const name of commands.available) {
+    const def = commands.get(name)!
+    lines.push(`| \`${name}\` | ${def.description} |`)
+  }
+
+  lines.push(
+    '',
+    '## Global Options',
+    '',
+    '| Option | Description |',
+    '|--------|-------------|',
+    '| `--help`, `-h` | Show help for a command |',
+    '| `--contentFolder` | Path to content folder (most commands) |',
+    '',
+    '## Getting Help',
+    '',
+    '```bash',
+    '# Show this overview',
+    'cnotes help',
+    '',
+    '# Show detailed help for a command',
+    'cnotes help serve',
+    'cnotes serve --help',
+    '```',
+  )
+
+  console.log(ui.markdown(lines.join('\n')))
+}
+
+commands.register('help', {
+  description: 'Show available commands',
+  help: `# cnotes help
+
+Show available commands and usage information.
+
+## Usage
+
+\`\`\`
+cnotes help [command]
+\`\`\`
+
+## Arguments
+
+| Argument | Description |
+|----------|-------------|
+| \`command\` | Show detailed help for a specific command |
+
+## Examples
+
+\`\`\`bash
+# Show all commands
+cnotes help
+
+# Show help for the serve command
+cnotes help serve
+\`\`\`
+`,
+  handler,
+})

package/src/cli/commands/index.ts

@@ -0,0 +1,17 @@
+// Side-effect imports register each command
+import './init.js'
+import './create.js'
+import './inspect.js'
+import './validate.js'
+import './export.js'
+import './action.js'
+import './summary.js'
+import './teach.js'
+import './extract.js'
+import './mcp.js'
+import './serve.js'
+import './console.js'
+import './help.js'
+import './text-search.js'
+import './search.js'
+import './embed.js'