contentbase 0.0.1 → 0.0.4

package/src/api/endpoints/documents.ts

@@ -0,0 +1,71 @@
+import { z } from 'zod'
+import matter from 'gray-matter'
+import { resolveModelDef } from '../helpers.js'
+
+export const path = '/api/documents'
+export const description = 'List or create documents'
+export const tags = ['documents']
+
+export const getSchema = z.object({
+  model: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+
+  let ids = collection.available as string[]
+
+  if (params.model) {
+    const def = resolveModelDef(collection, params.model)
+    if (!def) {
+      ctx.response.status(400)
+      return { error: `Unknown model: ${params.model}` }
+    }
+    const prefix = (def as any).prefix + '/'
+    ids = ids.filter((id: string) => id.startsWith(prefix))
+  }
+
+  return ids.map((id: string) => {
+    const doc = collection.document(id)
+    return { id, title: doc.title, meta: doc.meta, size: doc.size, createdAt: doc.createdAt, updatedAt: doc.updatedAt }
+  })
+}
+
+export const postSchema = z.object({
+  pathId: z.string(),
+  title: z.string(),
+  meta: z.record(z.string(), z.any()).optional(),
+  model: z.string().optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  if (ctx.container._contentbaseReadOnly) {
+    ctx.response.status(403)
+    return { error: 'Server is running in read-only mode' }
+  }
+
+  const collection = ctx.container._contentbaseCollection
+
+  if (collection.available.includes(params.pathId)) {
+    ctx.response.status(409)
+    return { error: `Document already exists: ${params.pathId}` }
+  }
+
+  const def = params.model
+    ? resolveModelDef(collection, params.model)
+    : collection.findModelDefinition(params.pathId)
+
+  const metaData = { ...((def as any)?.defaults || {}), ...(params.meta || {}) }
+
+  const sectionHeadings = def
+    ? Object.values((def as any).sections || {}).map((s: any) => `## ${s.heading}\n\n`)
+    : []
+
+  const body = [`# ${params.title}`, '', ...sectionHeadings].join('\n')
+  const content = matter.stringify(body, metaData)
+
+  await collection.saveItem(params.pathId, { content })
+
+  ctx.response.status(201)
+  return { created: params.pathId, model: def ? (def as any).name : null, meta: metaData }
+}

package/src/api/endpoints/inspect.ts

@@ -0,0 +1,16 @@
+import { buildSchemaJSON } from '../helpers.js'
+
+export const path = '/api/inspect'
+export const description = 'Collection overview — models, document count, actions'
+export const tags = ['collection']
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const schema = buildSchemaJSON(collection)
+  return {
+    rootPath: collection.rootPath,
+    documentCount: collection.available.length,
+    models: Object.values(schema),
+    actions: collection.availableActions,
+  }
+}

package/src/api/endpoints/models.ts

@@ -0,0 +1,10 @@
+import { buildSchemaJSON } from '../helpers.js'
+
+export const path = '/api/models'
+export const description = 'All model definitions with schemas, sections, and relationships'
+export const tags = ['collection']
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  return buildSchemaJSON(collection)
+}

package/src/api/endpoints/query.ts

@@ -0,0 +1,88 @@
+import { z } from 'zod'
+import { resolveModelDef } from '../helpers.js'
+import { queryDSLSchema, executeQueryDSL } from '../../query/query-dsl.js'
+
+export const path = '/api/query'
+export const description = 'Query model instances with filtering'
+export const tags = ['query']
+
+export const getSchema = z.object({
+  model: z.string(),
+  where: z.string().optional(),
+  select: z.string().optional(),
+  related: z.string().optional(),
+})
+
+export const postSchema = queryDSLSchema
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+
+  const def = resolveModelDef(collection, params.model)
+  if (!def) {
+    ctx.response.status(400)
+    return { error: `Unknown model: ${params.model}` }
+  }
+
+  let q = collection.query(def)
+
+  if (params.where) {
+    let conditions: any[]
+    try {
+      conditions = JSON.parse(params.where)
+    } catch {
+      ctx.response.status(400)
+      return { error: 'Invalid JSON in where parameter' }
+    }
+
+    for (const condition of conditions) {
+      const { path: fieldPath, operator = 'eq', value } = condition
+      switch (operator) {
+        case 'eq': q = q.where(fieldPath, value); break
+        case 'in': q = q.whereIn(fieldPath, value); break
+        case 'notIn': q = q.whereNotIn(fieldPath, value); break
+        case 'gt': q = q.whereGt(fieldPath, value); break
+        case 'lt': q = q.whereLt(fieldPath, value); break
+        case 'gte': q = q.whereGte(fieldPath, value); break
+        case 'lte': q = q.whereLte(fieldPath, value); break
+        case 'contains': q = q.whereContains(fieldPath, value); break
+        case 'startsWith': q = q.whereStartsWith(fieldPath, value); break
+        case 'endsWith': q = q.whereEndsWith(fieldPath, value); break
+        case 'regex': q = q.whereRegex(fieldPath, value); break
+        case 'exists': q = q.whereExists(fieldPath); break
+        case 'notExists': q = q.whereNotExists(fieldPath); break
+      }
+    }
+  }
+
+  const results = await q.fetchAll()
+  const selectFields = params.select ? params.select.split(',').map((s: string) => s.trim()) : null
+  const relatedFields = params.related ? params.related.split(',').map((s: string) => s.trim()) : undefined
+
+  return results.map((instance: any) => {
+    const json = instance.toJSON({ related: relatedFields })
+    if (selectFields && selectFields.length > 0) {
+      const filtered: Record<string, any> = {}
+      for (const key of selectFields) {
+        if (key in json) filtered[key] = json[key]
+        else if (key.startsWith('meta.') && json.meta) {
+          filtered[key] = json.meta[key.slice(5)]
+        }
+      }
+      return filtered
+    }
+    return json
+  })
+}
+
+export async function post(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+
+  try {
+    const dsl = queryDSLSchema.parse(params)
+    return await executeQueryDSL(collection, dsl)
+  } catch (error: any) {
+    ctx.response.status(400)
+    return { error: error.message }
+  }
+}

package/src/api/endpoints/root.ts

@@ -0,0 +1,7 @@
+export const path = '/'
+export const description = 'Redirect to docs table of contents when no static index.html exists'
+export const tags = ['docs']
+
+export async function get(_params: any, ctx: any) {
+  ctx.response.redirect('/docs/')
+}

package/src/api/endpoints/search-reindex.ts

@@ -0,0 +1,65 @@
+import { z } from 'zod'
+import pathModule from 'node:path'
+
+export const path = '/api/search/reindex'
+export const description = 'Trigger search index rebuild'
+export const tags = ['mutation']
+
+export const postSchema = z.object({
+  pathIds: z.array(z.string()).optional(),
+  force: z.boolean().optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  if (!ctx.container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(ctx.container)
+  }
+
+  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
+  const ss = ctx.container.feature('semanticSearch', { dbPath })
+  await ss.initDb()
+
+  if (params.pathIds) {
+    await ss.reindex(params.pathIds)
+  } else if (params.force) {
+    await ss.reindex()
+  }
+
+  // Collect and re-index documents
+  const docs: any[] = []
+  const targetIds = params.pathIds || collection.available
+
+  for (const pathId of targetIds) {
+    if (!collection.available.includes(pathId)) continue
+    const doc = collection.document(pathId)
+    const modelDef = collection.findModelDefinition(pathId)
+
+    docs.push({
+      pathId,
+      model: modelDef?.name ?? undefined,
+      title: doc.title,
+      slug: doc.slug,
+      meta: doc.meta,
+      content: doc.content,
+    })
+  }
+
+  const toIndex = params.force ? docs : docs.filter((d: any) => ss.needsReindex(d))
+
+  if (toIndex.length > 0) {
+    await ss.indexDocuments(toIndex)
+  }
+
+  ss.removeStale(collection.available)
+
+  const stats = ss.getStats()
+  return {
+    reindexed: toIndex.length,
+    total: docs.length,
+    ...stats,
+  }
+}

package/src/api/endpoints/search-status.ts

@@ -0,0 +1,55 @@
+import { z } from 'zod'
+import pathModule from 'node:path'
+import { existsSync, readdirSync } from 'node:fs'
+
+export const path = '/api/search/status'
+export const description = 'Search index health and statistics'
+export const tags = ['query']
+
+export const getSchema = z.object({})
+
+export async function get(_params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+  const dbDir = pathModule.join(rootPath, '.contentbase')
+
+  const hasIndex = existsSync(dbDir) && (() => {
+    try {
+      const files = readdirSync(dbDir) as string[]
+      return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
+    } catch {
+      return false
+    }
+  })()
+
+  if (!hasIndex) {
+    return {
+      exists: false,
+      documentCount: 0,
+      chunkCount: 0,
+      embeddingCount: 0,
+      lastIndexedAt: null,
+      provider: null,
+      model: null,
+      dimensions: 0,
+      dbSizeBytes: 0,
+      collectionDocumentCount: collection.available.length,
+    }
+  }
+
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  if (!ctx.container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(ctx.container)
+  }
+
+  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
+  const ss = ctx.container.feature('semanticSearch', { dbPath })
+  await ss.initDb()
+
+  const stats = ss.getStats()
+  return {
+    exists: true,
+    ...stats,
+    collectionDocumentCount: collection.available.length,
+  }
+}

package/src/api/endpoints/search.ts

@@ -0,0 +1,104 @@
+import { z } from 'zod'
+import pathModule from 'node:path'
+import { existsSync, readdirSync } from 'node:fs'
+
+export const path = '/api/search'
+export const description = 'Search across collection documents using keyword, semantic, or hybrid modes'
+export const tags = ['query']
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function hasSearchIndex(rootPath: string): boolean {
+  const dbDir = pathModule.join(rootPath, '.contentbase')
+  if (!existsSync(dbDir)) return false
+  try {
+    const files = readdirSync(dbDir) as string[]
+    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
+  } catch {
+    return false
+  }
+}
+
+let _semanticSearch: any = null
+
+async function getSemanticSearch(container: any, rootPath: string) {
+  if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
+
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  if (!container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(container)
+  }
+
+  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
+  _semanticSearch = container.feature('semanticSearch', { dbPath })
+  await _semanticSearch.initDb()
+  return _semanticSearch
+}
+
+async function doSearch(ss: any, query: string, mode: string, options: any) {
+  switch (mode) {
+    case 'keyword':
+      return ss.search(query, options)
+    case 'vector':
+      return ss.vectorSearch(query, options)
+    case 'hybrid':
+    default:
+      return ss.hybridSearch(query, options)
+  }
+}
+
+// ── GET /api/search ──────────────────────────────────────────────────
+
+export const getSchema = z.object({
+  q: z.string(),
+  mode: z.string().optional(),
+  model: z.string().optional(),
+  limit: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const limit = params.limit ? parseInt(params.limit, 10) : 10
+  const searchOptions = { limit, model: params.model }
+
+  return doSearch(ss, params.q, mode, searchOptions)
+}
+
+// ── POST /api/search ─────────────────────────────────────────────────
+
+export const postSchema = z.object({
+  query: z.string(),
+  mode: z.enum(['hybrid', 'keyword', 'vector']).optional(),
+  model: z.string().optional(),
+  limit: z.number().optional(),
+  where: z.record(z.string(), z.any()).optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const searchOptions = {
+    limit: params.limit || 10,
+    model: params.model,
+    where: params.where,
+  }
+
+  return doSearch(ss, params.query, mode, searchOptions)
+}

package/src/api/endpoints/semantic-search.ts

@@ -0,0 +1,120 @@
+import { z } from 'zod'
+import pathModule from 'node:path'
+import { existsSync, readdirSync } from 'node:fs'
+
+export const path = '/api/semantic-search'
+export const description = 'Semantic search across collection documents'
+export const tags = ['query']
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+function hasSearchIndex(rootPath: string): boolean {
+  const dbDir = pathModule.join(rootPath, '.contentbase')
+  if (!existsSync(dbDir)) return false
+  try {
+    const files = readdirSync(dbDir) as string[]
+    return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
+  } catch {
+    return false
+  }
+}
+
+let _semanticSearch: any = null
+
+async function getSemanticSearch(container: any, rootPath: string) {
+  if (_semanticSearch?.state?.get('dbReady')) return _semanticSearch
+
+  const { SemanticSearch } = await import('@soederpop/luca/agi')
+  if (!container.features.available.includes('semanticSearch')) {
+    SemanticSearch.attach(container)
+  }
+
+  const dbPath = pathModule.join(rootPath, '.contentbase/search.sqlite')
+  _semanticSearch = container.feature('semanticSearch', { dbPath })
+  await _semanticSearch.initDb()
+  return _semanticSearch
+}
+
+// ── GET /api/semantic-search ─────────────────────────────────────────
+
+export const getSchema = z.object({
+  q: z.string(),
+  mode: z.string().optional(),
+  model: z.string().optional(),
+  limit: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const limit = params.limit ? parseInt(params.limit, 10) : 10
+  const searchOptions = { limit, model: params.model }
+
+  let results: any[]
+  switch (mode) {
+    case 'keyword':
+      results = await ss.search(params.q, searchOptions)
+      break
+    case 'vector':
+      results = await ss.vectorSearch(params.q, searchOptions)
+      break
+    case 'hybrid':
+    default:
+      results = await ss.hybridSearch(params.q, searchOptions)
+      break
+  }
+
+  return results
+}
+
+// ── POST /api/semantic-search ────────────────────────────────────────
+
+export const postSchema = z.object({
+  query: z.string(),
+  mode: z.enum(['hybrid', 'keyword', 'vector']).optional(),
+  model: z.string().optional(),
+  limit: z.number().optional(),
+  where: z.record(z.string(), z.any()).optional(),
+})
+
+export async function post(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const rootPath = collection.rootPath
+
+  if (!hasSearchIndex(rootPath)) {
+    ctx.response.status(400)
+    return { error: 'No search index found. Run: cnotes embed' }
+  }
+
+  const ss = await getSemanticSearch(ctx.container, rootPath)
+  const mode = params.mode || 'hybrid'
+  const searchOptions = {
+    limit: params.limit || 10,
+    model: params.model,
+    where: params.where,
+  }
+
+  let results: any[]
+  switch (mode) {
+    case 'keyword':
+      results = await ss.search(params.query, searchOptions)
+      break
+    case 'vector':
+      results = await ss.vectorSearch(params.query, searchOptions)
+      break
+    case 'hybrid':
+    default:
+      results = await ss.hybridSearch(params.query, searchOptions)
+      break
+  }
+
+  return results
+}

package/src/api/endpoints/text-search.ts

@@ -0,0 +1,63 @@
+import { z } from 'zod'
+
+export const path = '/api/text-search'
+export const description = 'Search file contents with pattern matching'
+export const tags = ['query']
+
+export const getSchema = z.object({
+  pattern: z.string(),
+  expanded: z.string().optional(),
+  include: z.string().optional(),
+  exclude: z.string().optional(),
+  ignoreCase: z.string().optional(),
+  maxResults: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+  const grep = ctx.container.feature('grep')
+  const searchPath = collection.rootPath
+
+  const expanded = params.expanded === 'true'
+  const ignoreCase = params.ignoreCase === 'true'
+  const maxResults = params.maxResults ? parseInt(params.maxResults, 10) : undefined
+
+  if (!expanded) {
+    const files = await grep.filesContaining(params.pattern, {
+      path: searchPath,
+      ignoreCase,
+      maxResults,
+      include: params.include,
+      exclude: params.exclude,
+    })
+
+    return { files, count: files.length }
+  }
+
+  const results = await grep.search({
+    pattern: params.pattern,
+    path: searchPath,
+    ignoreCase,
+    maxResults,
+    include: params.include,
+    exclude: params.exclude,
+  })
+
+  // Group by file
+  const grouped = new Map<string, Array<{ line: number; column?: number; content: string }>>()
+  for (const match of results) {
+    if (!grouped.has(match.file)) grouped.set(match.file, [])
+    grouped.get(match.file)!.push({
+      line: match.line,
+      column: match.column,
+      content: match.content,
+    })
+  }
+
+  const files = Array.from(grouped.entries()).map(([file, matches]) => ({
+    file,
+    matches,
+  }))
+
+  return { files, count: files.length }
+}

package/src/api/endpoints/validate.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod'
+import { validateDocument } from '../../index.js'
+import { resolveModelDef } from '../helpers.js'
+
+export const path = '/api/validate'
+export const description = 'Validate a document against its model schema'
+export const tags = ['validation']
+
+export const getSchema = z.object({
+  pathId: z.string(),
+  model: z.string().optional(),
+})
+
+export async function get(params: any, ctx: any) {
+  const collection = ctx.container._contentbaseCollection
+
+  if (!collection.available.includes(params.pathId)) {
+    ctx.response.status(404)
+    return { error: `Document not found: ${params.pathId}` }
+  }
+
+  const doc = collection.document(params.pathId)
+
+  const def = params.model
+    ? resolveModelDef(collection, params.model)
+    : collection.findModelDefinition(params.pathId)
+
+  if (!def) {
+    ctx.response.status(400)
+    return { error: `No model definition found for ${params.pathId}` }
+  }
+
+  return validateDocument(doc, def)
+}