contentbase 0.2.0 → 0.4.0

package/src/mcp/tools/mutation.ts

@@ -0,0 +1,176 @@
+import { z } from 'zod'
+import matter from 'gray-matter'
+import { errorResult, textResult } from '../helpers.js'
+import { resolveModelDef } from '../../cli/commands/api/helpers.js'
+import { validateDocument } from '../../index.js'
+
+export function registerMutationTools(mcpServer: any, collection: any) {
+  const modelDefs = collection.modelDefinitions as any[]
+
+  mcpServer.tool('validate', {
+    description: 'Validate a document against its model schema. Returns validation result with any errors. **ALWAYS call after create/update operations** to confirm the document conforms to its model.',
+    schema: z.object({
+      pathId: z.string().describe('Document path ID'),
+      model: z.string().optional().describe('Model name (auto-detected if omitted)'),
+    }),
+    handler: (args: { pathId: string; model?: string }) => {
+      const doc = collection.document(args.pathId)
+      if (!doc) return errorResult(`Document not found: ${args.pathId}`)
+
+      const def = args.model
+        ? resolveModelDef(collection, args.model)
+        : collection.findModelDefinition(args.pathId)
+
+      if (!def) {
+        return errorResult(`No model definition found for ${args.pathId}. Specify one with the model parameter.`)
+      }
+
+      const result = validateDocument(doc, def)
+      return textResult(JSON.stringify(result, null, 2))
+    },
+  })
+
+  mcpServer.tool('create_document', {
+    description: '**ALWAYS use this instead of writing markdown files directly.** Creates a new document with proper scaffolding from a model definition — generates correct frontmatter defaults and section headings. Call `validate` after creation.',
+    schema: z.object({
+      pathId: z.string().describe('Path ID for the new document (e.g. "epics/my-new-epic")'),
+      title: z.string().describe('Document title (used as the H1 heading)'),
+      meta: z.record(z.string(), z.any()).optional().describe('Frontmatter fields to set'),
+      model: z.string().optional().describe('Model name (auto-detected from pathId prefix if omitted)'),
+    }),
+    handler: async (args: { pathId: string; title: string; meta?: Record<string, any>; model?: string }) => {
+      if (collection.available.includes(args.pathId)) {
+        return errorResult(`Document already exists: ${args.pathId}`)
+      }
+
+      const def = args.model
+        ? resolveModelDef(collection, args.model)
+        : collection.findModelDefinition(args.pathId)
+
+      const metaData = { ...((def as any)?.defaults || {}), ...(args.meta || {}) }
+
+      const sectionHeadings = def
+        ? Object.values((def as any).sections || {}).map((s: any) => `## ${s.heading}\n\n`)
+        : []
+
+      const body = [
+        `# ${args.title}`,
+        '',
+        ...sectionHeadings,
+      ].join('\n')
+
+      const content = matter.stringify(body, metaData)
+
+      await collection.saveItem(args.pathId, { content })
+      await collection.load({ refresh: true })
+
+      return textResult(JSON.stringify({
+        created: args.pathId,
+        model: def ? (def as any).name : null,
+        meta: metaData,
+      }, null, 2))
+    },
+  })
+
+  mcpServer.tool('update_document', {
+    description: 'Update a document\'s frontmatter and/or replace its entire content body. Use for frontmatter changes. For section-level edits, prefer `update_section` instead. Call `validate` after.',
+    schema: z.object({
+      pathId: z.string().describe('Document path ID'),
+      meta: z.record(z.string(), z.any()).optional().describe('Frontmatter fields to merge (existing fields are preserved unless overridden)'),
+      content: z.string().optional().describe('New markdown content body (replaces everything after frontmatter)'),
+    }),
+    handler: async (args: { pathId: string; meta?: Record<string, any>; content?: string }) => {
+      const doc = collection.document(args.pathId)
+      if (!doc) return errorResult(`Document not found: ${args.pathId}`)
+
+      const currentMeta = { ...doc.meta }
+      const newMeta = args.meta ? { ...currentMeta, ...args.meta } : currentMeta
+      const newContent = args.content ?? doc.content
+
+      const fullContent = matter.stringify(newContent, newMeta)
+      await collection.saveItem(args.pathId, { content: fullContent })
+      await collection.load({ refresh: true })
+
+      return textResult(JSON.stringify({
+        updated: args.pathId,
+        meta: newMeta,
+      }, null, 2))
+    },
+  })
+
+  mcpServer.tool('update_section', {
+    description: 'Preferred way to edit document content. Surgically edit a specific section — replace, append, or remove. Target a section by its heading name. Call `validate` after.',
+    schema: z.object({
+      pathId: z.string().describe('Document path ID'),
+      heading: z.string().describe('Section heading text to target (e.g. "Overview", "Requirements")'),
+      action: z.enum(['replace', 'append', 'remove']).describe('What to do with the section'),
+      content: z.string().optional().describe('New content (required for replace/append, ignored for remove)'),
+    }),
+    handler: async (args: { pathId: string; heading: string; action: 'replace' | 'append' | 'remove'; content?: string }) => {
+      let doc = collection.document(args.pathId)
+      if (!doc) return errorResult(`Document not found: ${args.pathId}`)
+
+      switch (args.action) {
+        case 'replace': {
+          if (!args.content) return errorResult('Content is required for replace action')
+          doc = doc.replaceSectionContent(args.heading, args.content)
+          break
+        }
+        case 'append': {
+          if (!args.content) return errorResult('Content is required for append action')
+          doc = doc.appendToSection(args.heading, args.content)
+          break
+        }
+        case 'remove': {
+          doc = doc.removeSection(args.heading)
+          break
+        }
+      }
+
+      const fullContent = matter.stringify(doc.content, doc.meta)
+      await collection.saveItem(args.pathId, { content: fullContent })
+      await collection.load({ refresh: true })
+
+      return textResult(JSON.stringify({
+        updated: args.pathId,
+        action: args.action,
+        heading: args.heading,
+      }, null, 2))
+    },
+  })
+
+  mcpServer.tool('delete_document', {
+    description: 'Delete a document from the collection permanently. Cannot be undone except through version control.',
+    schema: z.object({
+      pathId: z.string().describe('Document path ID to delete'),
+    }),
+    handler: async (args: { pathId: string }) => {
+      if (!collection.available.includes(args.pathId)) {
+        return errorResult(`Document not found: ${args.pathId}`)
+      }
+
+      await collection.deleteItem(args.pathId)
+      await collection.load({ refresh: true })
+      return textResult(JSON.stringify({ deleted: args.pathId }, null, 2))
+    },
+  })
+
+  mcpServer.tool('run_action', {
+    description: 'Execute a registered collection action by name.',
+    schema: z.object({
+      name: z.string().describe('Action name'),
+      args: z.array(z.any()).optional().describe('Arguments to pass to the action'),
+    }),
+    handler: async (toolArgs: { name: string; args?: any[] }) => {
+      if (!collection.availableActions.includes(toolArgs.name)) {
+        return errorResult(
+          `Unknown action: ${toolArgs.name}. Available: ${collection.availableActions.join(', ') || '(none)'}`,
+        )
+      }
+
+      const result = await collection.runAction(toolArgs.name, ...(toolArgs.args || []))
+      const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2)
+      return textResult(text)
+    },
+  })
+}

package/src/mcp/tools/query.ts

@@ -0,0 +1,141 @@
+import { z } from 'zod'
+import { errorResult, textResult } from '../helpers.js'
+import { generateReadMe } from '../readme.js'
+import { generateModelInfo } from '../model-info.js'
+import { resolveModelDef, buildSchemaJSON } from '../../cli/commands/api/helpers.js'
+import { queryDSLSchema, executeQueryDSL } from '../../query/query-dsl.js'
+
+export function registerQueryTools(mcpServer: any, collection: any) {
+  const modelDefs = collection.modelDefinitions as any[]
+
+  // -- read_me: entry-point guidance for AI agents --
+  const readMeContent = generateReadMe(collection, modelDefs)
+
+  mcpServer.tool('read_me', {
+    description: [
+      'Returns the content collection guide. Call this BEFORE working with any documents.',
+      'Contains model definitions, available tools, query syntax, and recommended workflow.',
+      'Call this at the start of every session to understand the collection structure.',
+    ].join('\n'),
+    schema: z.object({}),
+    handler: () => textResult(readMeContent),
+  })
+
+  mcpServer.tool('inspect', {
+    description: 'Overview of the collection — registered models, document count, available actions. Call `read_me` first if this is your first interaction.',
+    schema: z.object({}),
+    handler: () => {
+      const schema = buildSchemaJSON(collection)
+      const overview = {
+        rootPath: collection.rootPath,
+        documentCount: collection.available.length,
+        models: Object.values(schema),
+        actions: collection.availableActions,
+      }
+      return textResult(JSON.stringify(overview, null, 2))
+    },
+  })
+
+  mcpServer.tool('get_model_info', {
+    description: 'Get detailed information about a single model — fields, sections, relationships, example document. Use when you need to understand a model before creating or editing its documents.',
+    schema: z.object({
+      model: z.string().describe('Model name or prefix'),
+    }),
+    handler: (args: { model: string }) => {
+      const def = resolveModelDef(collection, args.model)
+      if (!def) {
+        return errorResult(`Unknown model: ${args.model}. Available: ${modelDefs.map((d: any) => d.name).join(', ')}`)
+      }
+      return textResult(generateModelInfo(collection, def))
+    },
+  })
+
+  mcpServer.tool('list_documents', {
+    description: 'List all document path IDs in the collection, optionally filtered by model name or prefix. The prefix before the slash indicates the model.',
+    schema: z.object({
+      model: z.string().optional().describe('Filter by model name or prefix'),
+    }),
+    handler: (args: { model?: string }) => {
+      let ids = collection.available
+
+      if (args.model) {
+        const def = resolveModelDef(collection, args.model)
+        if (def) {
+          const prefix = (def as any).prefix + '/'
+          ids = ids.filter((id: string) => id.startsWith(prefix))
+        } else {
+          return errorResult(`Unknown model: ${args.model}. Available: ${modelDefs.map((d: any) => d.name).join(', ')}`)
+        }
+      }
+
+      return textResult(JSON.stringify(ids, null, 2))
+    },
+  })
+
+  mcpServer.tool('query', {
+    description: [
+      'Query typed model instances with MongoDB-style filtering. See `read_me` output for full syntax reference.',
+      'Where clause: keys are dot-notation paths, values are literals (implies $eq),',
+      'arrays (implies $in), or operator objects like { "$gt": 5 }.',
+      'Operators: $eq, $neq, $in, $notIn, $gt, $lt, $gte, $lte,',
+      '$contains, $startsWith, $endsWith, $regex, $exists.',
+      'Supports sort, limit, offset, select, and method (fetchAll/first/last/count).',
+    ].join(' '),
+    schema: z.object({
+      model: z.string().describe('Model name to query'),
+      where: z.any().optional().describe(
+        'MongoDB-style where clause. Keys are field paths, values are literals (implicit $eq), arrays (implicit $in), or operator objects like { "$gt": 5 }. Also accepts legacy array format for backward compat.',
+      ),
+      sort: z.record(z.string(), z.enum(['asc', 'desc'])).optional().describe(
+        'Sort specification, e.g. { "meta.priority": "desc" }',
+      ),
+      select: z.array(z.string()).optional().describe('Fields to include in output (default: all)'),
+      related: z.array(z.string()).optional().describe('Relationship names to include in results (e.g. ["plans", "goal"])'),
+      scopes: z.array(z.string()).optional().describe('Named scopes to apply before filtering'),
+      limit: z.number().optional().describe('Maximum results to return'),
+      offset: z.number().optional().describe('Number of results to skip'),
+      method: z.enum(['fetchAll', 'first', 'last', 'count']).optional().describe(
+        'Terminal operation (default: fetchAll)',
+      ),
+    }),
+    handler: async (args: any) => {
+      try {
+        // Backward compat: convert legacy array-style where to MongoDB-style
+        let whereClause = args.where
+        if (Array.isArray(whereClause)) {
+          const converted: Record<string, unknown> = {}
+          for (const cond of whereClause) {
+            const op = cond.operator || 'eq'
+            if (op === 'eq') {
+              converted[cond.path] = cond.value
+            } else if (op === 'notExists') {
+              converted[cond.path] = { $exists: false }
+            } else if (op === 'exists') {
+              converted[cond.path] = { $exists: true }
+            } else {
+              converted[cond.path] = { [`$${op}`]: cond.value }
+            }
+          }
+          whereClause = converted
+        }
+
+        const dsl = queryDSLSchema.parse({
+          model: args.model,
+          where: whereClause,
+          sort: args.sort,
+          select: args.select,
+          related: args.related,
+          scopes: args.scopes,
+          limit: args.limit,
+          offset: args.offset,
+          method: args.method,
+        })
+
+        const result = await executeQueryDSL(collection, dsl)
+        return textResult(JSON.stringify(result, null, 2))
+      } catch (error: any) {
+        return errorResult(error.message)
+      }
+    },
+  })
+}

package/src/mcp/tools/search.ts

@@ -0,0 +1,180 @@
+import { z } from 'zod'
+import { errorResult, textResult } from '../helpers.js'
+import { resolveModelDef } from '../../cli/commands/api/helpers.js'
+import { getInitializedSemanticSearch, hasSearchIndex as searchIndexExists } from '../../search/luca-semantic-search.js'
+
+export function registerSearchTools(mcpServer: any, collection: any, container: any) {
+  let _semanticSearch: any = null
+
+  function hasSearchIndex(): boolean {
+    return searchIndexExists(collection.rootPath)
+  }
+
+  async function getSemanticSearch() {
+    if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
+
+    _semanticSearch = await getInitializedSemanticSearch(container, collection.rootPath)
+    return _semanticSearch
+  }
+
+  mcpServer.tool('search_content', {
+    description: 'Full-text regex search across all document content. Returns matching document IDs with context. Searches document body text, not metadata — for metadata filtering, use `query` instead.',
+    schema: z.object({
+      pattern: z.string().describe('Regex pattern to search for'),
+      model: z.string().optional().describe('Limit search to a specific model'),
+      caseSensitive: z.boolean().default(false).describe('Case-sensitive matching'),
+    }),
+    handler: (args: { pattern: string; model?: string; caseSensitive: boolean }) => {
+      const flags = args.caseSensitive ? 'g' : 'gi'
+      let regex: RegExp
+      try {
+        regex = new RegExp(args.pattern, flags)
+      } catch (e: any) {
+        return errorResult(`Invalid regex: ${e.message}`)
+      }
+
+      let ids = collection.available
+      if (args.model) {
+        const def = resolveModelDef(collection, args.model)
+        if (def) {
+          const prefix = (def as any).prefix + '/'
+          ids = ids.filter((id: string) => id.startsWith(prefix))
+        }
+      }
+
+      const results: Array<{ pathId: string; matches: string[] }> = []
+
+      for (const pathId of ids) {
+        const doc = collection.document(pathId)
+        const content = doc.content
+        const matches: string[] = []
+
+        for (const line of content.split('\n')) {
+          if (regex.test(line)) {
+            matches.push(line.trim())
+          }
+          regex.lastIndex = 0
+        }
+
+        if (matches.length > 0) {
+          results.push({ pathId, matches: matches.slice(0, 10) })
+        }
+      }
+
+      return textResult(JSON.stringify(results, null, 2))
+    },
+  })
+
+  mcpServer.tool('text_search', {
+    description: 'Search file contents with pattern matching using ripgrep. Returns distinct file matches by default, or line-level detail with expanded=true.',
+    schema: z.object({
+      pattern: z.string().describe('Text or regex pattern to search for'),
+      expanded: z.boolean().default(false).describe('Return line-level matches instead of just file paths'),
+      include: z.string().optional().describe('Glob filter (e.g. "*.md")'),
+      exclude: z.string().optional().describe('Glob filter (e.g. "node_modules")'),
+      ignoreCase: z.boolean().default(false).describe('Case insensitive search'),
+      maxResults: z.number().optional().describe('Limit number of results'),
+    }),
+    handler: async (args: any) => {
+      const grep = container.feature('grep')
+      const searchPath = collection.rootPath
+
+      const grepOpts: any = {
+        path: searchPath,
+        ignoreCase: args.ignoreCase,
+        maxResults: args.maxResults,
+        include: args.include,
+        exclude: args.exclude,
+      }
+
+      if (!args.expanded) {
+        const files = await grep.filesContaining(args.pattern, grepOpts)
+        return textResult(JSON.stringify({ files, count: files.length }, null, 2))
+      }
+
+      const results = await grep.search({ ...grepOpts, pattern: args.pattern })
+      const grouped = new Map<string, Array<{ line: number; column?: number; content: string }>>()
+      for (const match of results) {
+        if (!grouped.has(match.file)) grouped.set(match.file, [])
+        grouped.get(match.file)!.push({ line: match.line, column: match.column, content: match.content })
+      }
+
+      const files = Array.from(grouped.entries()).map(([file, matches]) => ({ file, matches }))
+      return textResult(JSON.stringify({ files, count: files.length }, null, 2))
+    },
+  })
+
+  mcpServer.tool('keyword_search', {
+    description: 'Fast keyword search using BM25 ranking. Best for exact terms, identifiers, and known phrases. Requires a search index — run `cnotes embed` first if not indexed.',
+    schema: z.object({
+      query: z.string().describe('Search query text'),
+      limit: z.number().optional().default(10).describe('Maximum results to return'),
+      model: z.string().optional().describe('Filter results to a specific model name'),
+    }),
+    handler: async (args: { query: string; limit: number; model?: string }) => {
+      if (!hasSearchIndex()) {
+        return errorResult('No search index found. Run: cnotes embed')
+      }
+      try {
+        const ss = await getSemanticSearch()
+        const results = await ss.search(args.query, {
+          limit: args.limit,
+          model: args.model,
+        })
+        return textResult(JSON.stringify(results, null, 2))
+      } catch (error: any) {
+        return errorResult(`Search failed: ${error.message}`)
+      }
+    },
+  })
+
+  mcpServer.tool('semantic_search', {
+    description: 'Search by meaning using vector embeddings. Finds conceptually related documents even without keyword matches. Requires a search index — run `cnotes embed` first if not indexed.',
+    schema: z.object({
+      query: z.string().describe('Search query text'),
+      limit: z.number().optional().default(10).describe('Maximum results to return'),
+      model: z.string().optional().describe('Filter results to a specific model name'),
+    }),
+    handler: async (args: { query: string; limit: number; model?: string }) => {
+      if (!hasSearchIndex()) {
+        return errorResult('No search index found. Run: cnotes embed')
+      }
+      try {
+        const ss = await getSemanticSearch()
+        const results = await ss.vectorSearch(args.query, {
+          limit: args.limit,
+          model: args.model,
+        })
+        return textResult(JSON.stringify(results, null, 2))
+      } catch (error: any) {
+        return errorResult(`Search failed: ${error.message}`)
+      }
+    },
+  })
+
+  mcpServer.tool('hybrid_search', {
+    description: 'Combined keyword + semantic search with score fusion. Best for general questions about the collection. Requires a search index — run `cnotes embed` first if not indexed.',
+    schema: z.object({
+      query: z.string().describe('Search query text'),
+      limit: z.number().optional().default(10).describe('Maximum results to return'),
+      model: z.string().optional().describe('Filter results to a specific model name'),
+      where: z.record(z.string(), z.any()).optional().describe('Metadata filters, e.g. {"status": "approved"}'),
+    }),
+    handler: async (args: { query: string; limit: number; model?: string; where?: Record<string, any> }) => {
+      if (!hasSearchIndex()) {
+        return errorResult('No search index found. Run: cnotes embed')
+      }
+      try {
+        const ss = await getSemanticSearch()
+        const results = await ss.hybridSearch(args.query, {
+          limit: args.limit,
+          model: args.model,
+          where: args.where,
+        })
+        return textResult(JSON.stringify(results, null, 2))
+      } catch (error: any) {
+        return errorResult(`Search failed: ${error.message}`)
+      }
+    },
+  })
+}

package/src/parse.ts

@@ -7,6 +7,8 @@ import { toString } from "mdast-util-to-string";
 import { AstQuery } from "./ast-query";
 import { NodeShortcuts } from "./node-shortcuts";
 import { stringifyAst } from "./utils/stringify-ast";
+import { stripMarkdown } from "./utils/strip-markdown";
+import type { StripMarkdownOptions } from "./utils/strip-markdown";
 import type { Root, Content, RootContent } from "mdast";
 
 const processor = unified().use(remarkParse).use(remarkGfm);
@@ -26,6 +28,8 @@ export interface ParsedDocument {
   title: string;
   /** Stringify an AST back to markdown */
   stringify(ast?: Root): string;
+  /** Strip markdown syntax, returning plain text */
+  stripMarkdown(options?: StripMarkdownOptions): string;
   /** Extract a section by heading text */
   extractSection(heading: string | Content): Content[];
   /** Get a queryable AstQuery scoped to a section */
@@ -107,6 +111,7 @@ export async function parse(input: string): Promise<ParsedDocument> {
     nodes,
     title,
     stringify: (tree: Root = ast) => stringifyAst(tree),
+    stripMarkdown: (options?: StripMarkdownOptions) => stripMarkdown(ast, options),
     extractSection,
     querySection,
   };

package/src/query/query-dsl.ts

@@ -122,7 +122,7 @@ export function parseWhereClause(
           );
         }
 
-        const operator = OPERATOR_MAP[opKey];
+        const operator = OPERATOR_MAP[opKey]!;
 
         // Regex length guard
         if (

package/src/relationships/has-many.ts

@@ -135,7 +135,7 @@ export class HasManyRelationship<
     // e.g. Project hasMany Plans → looks for meta.project on Plan documents.
     const fk = this.#definition.foreignKey || this.#inferForeignKey();
     const slug = this.#document.slug;
-    const parentPrefix = this.#document.id.split("/")[0];
+    const parentPrefix = this.#document.id.split("/")[0]!;
     const idSegment = this.#document.id.slice(parentPrefix.length + 1);
     const prefix = targetDef.prefix;
     const results: InferModelInstance<TTarget>[] = [];
@@ -162,7 +162,7 @@ export class HasManyRelationship<
    * e.g. if the parent is in "projects/", the FK is "project" (singularized, lowercased).
    */
   #inferForeignKey(): string {
-    const parentPrefix = this.#document.id.split("/")[0];
+    const parentPrefix = this.#document.id.split("/")[0]!;
     // Simple singularize: strip trailing "s"
     return parentPrefix.replace(/s$/, "");
   }

package/src/search/document-inputs.ts

@@ -0,0 +1,85 @@
+export interface SearchDocumentSection {
+  heading: string
+  headingPath: string
+  content: string
+  level: number
+}
+
+export interface SearchDocumentInput {
+  pathId: string
+  model?: string
+  title?: string
+  slug?: string
+  meta?: Record<string, unknown>
+  content: string
+  sections?: SearchDocumentSection[]
+}
+
+export interface SearchCollectionLike {
+  available: string[]
+  document(pathId: string): {
+    title?: string
+    slug?: string
+    meta?: Record<string, unknown>
+    content: string
+  }
+  findModelDefinition?(pathId: string): { name?: string } | undefined
+}
+
+export function collectDocumentInputs(collection: SearchCollectionLike): SearchDocumentInput[] {
+  const inputs: SearchDocumentInput[] = []
+
+  for (const pathId of collection.available) {
+    const doc = collection.document(pathId)
+    const modelDef = collection.findModelDefinition?.(pathId)
+    const sections = collectH2Sections(doc.content)
+
+    inputs.push({
+      pathId,
+      model: modelDef?.name ?? undefined,
+      title: doc.title,
+      slug: doc.slug,
+      meta: doc.meta,
+      content: doc.content,
+      sections: sections.length > 0 ? sections : undefined,
+    })
+  }
+
+  return inputs
+}
+
+export function collectH2Sections(content: string): SearchDocumentSection[] {
+  const sections: SearchDocumentSection[] = []
+  const lines = content.split('\n')
+  let currentHeading: string | null = null
+  let currentContent: string[] = []
+
+  for (const line of lines) {
+    const h2Match = line.match(/^## (.+)/)
+    if (h2Match) {
+      if (currentHeading) {
+        sections.push({
+          heading: currentHeading,
+          headingPath: currentHeading,
+          content: currentContent.join('\n').trim(),
+          level: 2,
+        })
+      }
+      currentHeading = h2Match[1].trim()
+      currentContent = []
+    } else if (currentHeading) {
+      currentContent.push(line)
+    }
+  }
+
+  if (currentHeading) {
+    sections.push({
+      heading: currentHeading,
+      headingPath: currentHeading,
+      content: currentContent.join('\n').trim(),
+      level: 2,
+    })
+  }
+
+  return sections
+}