@nixxie-cms/ai-rag 1.0.0

package/src/admin-page.ts

@@ -0,0 +1,135 @@
+import type { AdminFileToWrite } from '@nixxie-cms/core/types'
+
+export type RagAdminPageOptions = {
+  /**
+   * Route the page is served at inside the Admin UI (also its filename under `pages/`).
+   * @default 'ai-rag'
+   */
+  route?: string
+  /**
+   * Base path of the HTTP routes the page talks to (must match `ragPlugin({ routes })`).
+   * @default '/api/ai-rag'
+   */
+  apiPath?: string
+  /** Heading shown at the top of the page. @default 'AI Assistant' */
+  title?: string
+}
+
+/** Source of the self-contained Admin UI page (plain React + fetch, no extra deps). */
+function pageSource(apiPath: string, title: string): string {
+  return `// Generated by @nixxie-cms/ai-rag — a chat + knowledge-base console for the Admin UI.
+import React, { useState } from 'react'
+
+const API = ${JSON.stringify(apiPath)}
+
+export default function AiRagPage() {
+  const [messages, setMessages] = useState([])
+  const [input, setInput] = useState('')
+  const [busy, setBusy] = useState(false)
+  const [docTitle, setDocTitle] = useState('')
+  const [docContent, setDocContent] = useState('')
+
+  async function send() {
+    const question = input.trim()
+    if (!question || busy) return
+    setInput('')
+    const next = [...messages, { role: 'user', content: question }]
+    setMessages(next)
+    setBusy(true)
+    try {
+      const res = await fetch(API + '/chat', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ messages: next, stream: false }),
+      })
+      const answer = await res.json()
+      const sources = (answer.sources || [])
+        .map(function (s, i) { return '[' + (i + 1) + '] ' + (s.title || s.source || s.documentId) })
+        .join('  ')
+      setMessages(function (m) {
+        return m.concat([{ role: 'assistant', content: answer.text + (sources ? '\\n\\nSources: ' + sources : '') }])
+      })
+    } catch (err) {
+      setMessages(function (m) { return m.concat([{ role: 'assistant', content: 'Error: ' + String(err) }]) })
+    } finally {
+      setBusy(false)
+    }
+  }
+
+  async function addDocument() {
+    if (!docContent.trim()) return
+    await fetch(API + '/documents', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ title: docTitle, content: docContent }),
+    })
+    setDocTitle('')
+    setDocContent('')
+    alert('Document added and queued for indexing.')
+  }
+
+  return (
+    React.createElement('div', { style: { maxWidth: 820, margin: '0 auto', padding: 24 } },
+      React.createElement('h1', { style: { fontSize: 24, fontWeight: 700, marginBottom: 16 } }, ${JSON.stringify(title)}),
+      React.createElement('div', { style: { border: '1px solid #e2e8f0', borderRadius: 8, padding: 16, minHeight: 240, marginBottom: 12 } },
+        messages.length === 0
+          ? React.createElement('p', { style: { color: '#94a3b8' } }, 'Ask the assistant a question about your knowledge base…')
+          : messages.map(function (m, i) {
+              return React.createElement('div', { key: i, style: { margin: '8px 0' } },
+                React.createElement('strong', null, m.role === 'user' ? 'You: ' : 'Assistant: '),
+                React.createElement('span', { style: { whiteSpace: 'pre-wrap' } }, m.content))
+            })
+      ),
+      React.createElement('div', { style: { display: 'flex', gap: 8, marginBottom: 32 } },
+        React.createElement('input', {
+          value: input,
+          onChange: function (e) { setInput(e.target.value) },
+          onKeyDown: function (e) { if (e.key === 'Enter') send() },
+          placeholder: 'Type your question…',
+          style: { flex: 1, padding: 8, borderRadius: 6, border: '1px solid #cbd5e1' },
+        }),
+        React.createElement('button', { onClick: send, disabled: busy, style: { padding: '8px 16px' } }, busy ? '…' : 'Send')
+      ),
+      React.createElement('h2', { style: { fontSize: 18, fontWeight: 600, marginBottom: 8 } }, 'Add to knowledge base'),
+      React.createElement('input', {
+        value: docTitle,
+        onChange: function (e) { setDocTitle(e.target.value) },
+        placeholder: 'Title',
+        style: { width: '100%', padding: 8, borderRadius: 6, border: '1px solid #cbd5e1', marginBottom: 8 },
+      }),
+      React.createElement('textarea', {
+        value: docContent,
+        onChange: function (e) { setDocContent(e.target.value) },
+        placeholder: 'Content the assistant should learn from…',
+        rows: 6,
+        style: { width: '100%', padding: 8, borderRadius: 6, border: '1px solid #cbd5e1', marginBottom: 8 },
+      }),
+      React.createElement('button', { onClick: addDocument, style: { padding: '8px 16px' } }, 'Add document')
+    )
+  )
+}
+`
+}
+
+/**
+ * Returns an `AdminFileToWrite` that adds a chat + knowledge-base console page to the
+ * Admin UI. Spread it into `ui.getAdditionalFiles`:
+ *
+ * @example
+ * import { ragAdminPage } from '@nixxie-cms/ai-rag'
+ * export default config({
+ *   ui: { getAdditionalFiles: [async () => [ragAdminPage()]] },
+ * })
+ *
+ * The page appears at `/<route>` (default `/ai-rag`) in the Admin UI.
+ */
+export function ragAdminPage(options: RagAdminPageOptions = {}): AdminFileToWrite {
+  const route = options.route ?? 'ai-rag'
+  const apiPath = (options.apiPath ?? '/api/ai-rag').replace(/\/$/, '')
+  const title = options.title ?? 'AI Assistant'
+  return {
+    mode: 'write',
+    src: pageSource(apiPath, title),
+    outputPath: `pages/${route}.js`,
+  }
+}

package/src/chunking.ts

@@ -0,0 +1,78 @@
+import type { RagChunkingConfig } from './types'
+
+const DEFAULTS = {
+  strategy: 'recursive' as const,
+  chunkSize: 1200,
+  chunkOverlap: 200,
+}
+
+/** Boundary separators tried in order, coarsest first (recursive strategy). */
+const SEPARATORS = ['\n\n', '\n', '. ', '! ', '? ', '; ', ', ', ' ']
+
+/**
+ * Split text into overlapping chunks. The recursive strategy prefers to cut on the
+ * coarsest natural boundary that keeps a chunk under `chunkSize`, falling back to finer
+ * boundaries (and finally a hard slice) so no chunk overruns the budget.
+ */
+export function chunkText(text: string, config: RagChunkingConfig = {}): string[] {
+  const strategy = config.strategy ?? DEFAULTS.strategy
+  const size = Math.max(1, config.chunkSize ?? DEFAULTS.chunkSize)
+  const overlap = Math.max(0, Math.min(config.chunkOverlap ?? DEFAULTS.chunkOverlap, size - 1))
+
+  const normalized = text.replace(/\r\n/g, '\n').trim()
+  if (!normalized) return []
+  if (normalized.length <= size) return [normalized]
+
+  const units =
+    strategy === 'sentence'
+      ? splitSentences(normalized)
+      : strategy === 'fixed'
+        ? hardSlice(normalized, size)
+        : recursiveSplit(normalized, size)
+
+  return packWithOverlap(units, size, overlap)
+}
+
+/** Greedily pack atomic units into chunks <= size, carrying `overlap` chars between them. */
+function packWithOverlap(units: string[], size: number, overlap: number): string[] {
+  const chunks: string[] = []
+  let current = ''
+  for (const unit of units) {
+    if (current && current.length + unit.length + 1 > size) {
+      chunks.push(current.trim())
+      current = overlap > 0 ? current.slice(Math.max(0, current.length - overlap)) : ''
+    }
+    current = current ? `${current} ${unit}`.trim() : unit
+    // A single oversized unit: hard-slice it.
+    while (current.length > size) {
+      chunks.push(current.slice(0, size).trim())
+      current = current.slice(size - overlap)
+    }
+  }
+  if (current.trim()) chunks.push(current.trim())
+  return chunks.filter(Boolean)
+}
+
+function recursiveSplit(text: string, size: number): string[] {
+  if (text.length <= size) return [text]
+  for (const sep of SEPARATORS) {
+    if (!text.includes(sep)) continue
+    const parts = text.split(sep).filter(Boolean)
+    if (parts.length < 2) continue
+    return parts.flatMap(p => (p.length > size ? recursiveSplit(p, size) : [p]))
+  }
+  return hardSlice(text, size)
+}
+
+function splitSentences(text: string): string[] {
+  return text
+    .split(/(?<=[.!?])\s+/)
+    .map(s => s.trim())
+    .filter(Boolean)
+}
+
+function hardSlice(text: string, size: number): string[] {
+  const out: string[] = []
+  for (let i = 0; i < text.length; i += size) out.push(text.slice(i, i + size))
+  return out
+}

package/src/collection.ts

@@ -0,0 +1,79 @@
+import { integer, json, select, text, timestamp } from '@nixxie-cms/core/fields'
+
+/**
+ * The knowledge-base source list the assistant is trained on. Users add rows here; each
+ * row's `content` is chunked, embedded and made retrievable. `ragPlugin()` adds this
+ * collection automatically, but you can register it yourself for full control over access.
+ *
+ * @example
+ * config({
+ *   collections: { KnowledgeBase: knowledgeBaseCollection(), ...collections },
+ * })
+ */
+export function knowledgeBaseCollection(): any {
+  return {
+    fields: {
+      title: text({ validation: { isRequired: true }, isIndexed: true }),
+      content: text({ ui: { displayMode: 'textarea' }, validation: { isRequired: true } }),
+      source: text(),
+      tags: json({ defaultValue: [] }),
+      metadata: json(),
+      status: select({
+        type: 'string',
+        options: [
+          { label: 'Pending', value: 'pending' },
+          { label: 'Indexing', value: 'indexing' },
+          { label: 'Indexed', value: 'indexed' },
+          { label: 'Error', value: 'error' },
+          { label: 'Disabled', value: 'disabled' },
+        ],
+        defaultValue: 'pending',
+        isIndexed: true,
+        ui: { displayMode: 'segmented-control' },
+      }),
+      chunkCount: integer({ defaultValue: 0 }),
+      error: text(),
+      indexedAt: timestamp(),
+      createdAt: timestamp({ defaultValue: { kind: 'now' }, db: { isNullable: false } }),
+      updatedAt: timestamp({ db: { updatedAt: true } }),
+    },
+    ui: {
+      labelField: 'title',
+      description: 'Knowledge base the AI assistant is trained on.',
+      listView: {
+        initialColumns: ['title', 'status', 'chunkCount', 'tags', 'updatedAt'],
+        initialSort: { field: 'updatedAt', direction: 'DESC' },
+      },
+    },
+  }
+}
+
+/**
+ * The chunk/embedding list backing retrieval. Managed entirely by ai-rag — rows are written
+ * during indexing and deleted with their document. Hidden from create/delete in the Admin UI.
+ * `ragPlugin()` adds it automatically.
+ */
+export function knowledgeChunkCollection(): any {
+  return {
+    fields: {
+      documentId: text({ validation: { isRequired: true }, isIndexed: true }),
+      content: text({ ui: { displayMode: 'textarea' } }),
+      embedding: json(),
+      title: text(),
+      source: text(),
+      tags: json({ defaultValue: [] }),
+      metadata: json(),
+      createdAt: timestamp({ defaultValue: { kind: 'now' }, db: { isNullable: false } }),
+    },
+    graphql: {
+      omit: { create: true, update: true, delete: true },
+    },
+    ui: {
+      isHidden: true,
+      labelField: 'title',
+      hideCreate: true,
+      hideDelete: true,
+      itemView: { defaultFieldMode: 'read' },
+    },
+  }
+}

package/src/express.ts

@@ -0,0 +1,212 @@
+import type { NixxieAiRagService, NixxieContext, NixxieRagStreamEvent } from '@nixxie-cms/core'
+
+export type AiRagRoutesOptions = {
+  /** The service. Defaults to `context.services.aiRag`. */
+  service?: NixxieAiRagService
+  /**
+   * Base path the routes mount under.
+   * @default '/api/ai-rag'
+   */
+  path?: string
+  /**
+   * Authorize a request before it reaches the assistant. Receives a request-bound context;
+   * returning false answers 401. Applied to every route.
+   * @default requires a session
+   */
+  isAuthorized?: (context: NixxieContext) => boolean | Promise<boolean>
+  /**
+   * Require authorization for read/chat routes too (not just mutations).
+   * @default true
+   */
+  protectReads?: boolean
+}
+
+function getService(context: NixxieContext, options: AiRagRoutesOptions): NixxieAiRagService {
+  const svc = options.service ?? context.services.aiRag
+  if (!svc) {
+    throw new Error(
+      '[@nixxie-cms/ai-rag] No aiRag service configured. Pass `service` or set `config({ aiRag })`.'
+    )
+  }
+  return svc
+}
+
+/** Read a JSON body whether or not a body parser ran upstream. */
+async function readJson(req: any): Promise<any> {
+  if (req.body && typeof req.body === 'object') return req.body
+  return new Promise(resolve => {
+    let raw = ''
+    req.on('data', (c: any) => (raw += c))
+    req.on('end', () => {
+      try {
+        resolve(raw ? JSON.parse(raw) : {})
+      } catch {
+        resolve({})
+      }
+    })
+    req.on('error', () => resolve({}))
+  })
+}
+
+function writeSse(res: any, event: NixxieRagStreamEvent): void {
+  const type = event.type
+  res.write(`event: ${type}\ndata: ${JSON.stringify(event)}\n\n`)
+}
+
+/**
+ * Mount the assistant's HTTP API on an Express app:
+ *
+ * - `POST <path>/chat` — body `{ messages?, question?, topK?, tags?, stream? }`. When
+ *   `stream` is true (or `Accept: text/event-stream`), responds with SSE: `sources`,
+ *   then `token` events, then a final `done` event carrying the full answer.
+ * - `GET <path>/documents` — list knowledge-base documents.
+ * - `POST <path>/documents` — add a document `{ content, title?, source?, tags? }`.
+ * - `DELETE <path>/documents/:id` — remove a document.
+ * - `POST <path>/reindex` — `{ force? }` → index stats.
+ *
+ * Call from `server.extendExpressApp`, or use `ragPlugin()` to wire it automatically.
+ */
+export function installAiRagRoutes(
+  app: any,
+  context: NixxieContext,
+  options: AiRagRoutesOptions = {}
+): void {
+  const path = (options.path ?? '/api/ai-rag').replace(/\/$/, '')
+  const protectReads = options.protectReads ?? true
+
+  const authorize = async (req: any, res: any): Promise<NixxieContext | null> => {
+    const requestContext = await context.withRequest(req, res)
+    const check = options.isAuthorized ?? ((c: NixxieContext) => !!c.session)
+    if (!(await check(requestContext))) {
+      res.statusCode = 401
+      res.end('Unauthorized')
+      return null
+    }
+    return requestContext
+  }
+
+  const wrap =
+    (handler: (req: any, res: any, ctx: NixxieContext) => Promise<void>, gated: boolean) =>
+    (req: any, res: any) => {
+      void (async () => {
+        const ctx = gated ? await authorize(req, res) : await context.withRequest(req, res)
+        if (!ctx) return
+        await handler(req, res, ctx)
+      })().catch((err: unknown) => {
+        console.error('[@nixxie-cms/ai-rag] Route error:', err)
+        if (!res.headersSent) {
+          res.statusCode = 500
+          res.setHeader('Content-Type', 'application/json')
+          res.end(JSON.stringify({ error: err instanceof Error ? err.message : 'Internal error' }))
+        } else {
+          res.end()
+        }
+      })
+    }
+
+  // ── Chat (streaming or one-shot) ──
+  app.post(
+    `${path}/chat`,
+    wrap(async (req, res, ctx) => {
+      const svc = getService(ctx, options)
+      const body = await readJson(req)
+      const messages =
+        Array.isArray(body.messages) && body.messages.length
+          ? body.messages
+          : body.question
+            ? [{ role: 'user', content: String(body.question) }]
+            : []
+      if (!messages.length) {
+        res.statusCode = 400
+        res.setHeader('Content-Type', 'application/json')
+        res.end(JSON.stringify({ error: 'Provide `messages` or `question`.' }))
+        return
+      }
+      const askOptions = {
+        topK: body.topK,
+        tags: body.tags,
+        temperature: body.temperature,
+        model: body.model,
+      }
+      const wantsStream =
+        body.stream === true || String(req.headers?.accept ?? '').includes('text/event-stream')
+
+      if (!wantsStream) {
+        const answer = await svc.chat(messages, askOptions)
+        res.setHeader('Content-Type', 'application/json')
+        res.end(JSON.stringify(answer))
+        return
+      }
+
+      res.setHeader('Content-Type', 'text/event-stream')
+      res.setHeader('Cache-Control', 'no-cache')
+      res.setHeader('Connection', 'keep-alive')
+      res.flushHeaders?.()
+      let aborted = false
+      req.on?.('close', () => (aborted = true))
+      for await (const event of svc.stream(messages, askOptions)) {
+        if (aborted) break
+        writeSse(res, event)
+      }
+      res.end()
+    }, protectReads)
+  )
+
+  // ── Documents CRUD ──
+  app.get(
+    `${path}/documents`,
+    wrap(async (req, res, ctx) => {
+      const svc = getService(ctx, options)
+      const take = req.query?.take ? Number(req.query.take) : undefined
+      const skip = req.query?.skip ? Number(req.query.skip) : undefined
+      const docs = await svc.listDocuments({ take, skip, search: req.query?.search })
+      res.setHeader('Content-Type', 'application/json')
+      res.end(JSON.stringify({ documents: docs }))
+    }, protectReads)
+  )
+
+  app.post(
+    `${path}/documents`,
+    wrap(async (req, res, ctx) => {
+      const svc = getService(ctx, options)
+      const body = await readJson(req)
+      if (!body.content) {
+        res.statusCode = 400
+        res.setHeader('Content-Type', 'application/json')
+        res.end(JSON.stringify({ error: '`content` is required.' }))
+        return
+      }
+      const doc = await svc.addDocument({
+        content: String(body.content),
+        title: body.title,
+        source: body.source,
+        tags: body.tags,
+        metadata: body.metadata,
+      })
+      res.statusCode = 201
+      res.setHeader('Content-Type', 'application/json')
+      res.end(JSON.stringify(doc))
+    }, true)
+  )
+
+  app.delete(
+    `${path}/documents/:id`,
+    wrap(async (req, res, ctx) => {
+      const svc = getService(ctx, options)
+      await svc.removeDocument(req.params.id)
+      res.setHeader('Content-Type', 'application/json')
+      res.end(JSON.stringify({ ok: true }))
+    }, true)
+  )
+
+  app.post(
+    `${path}/reindex`,
+    wrap(async (req, res, ctx) => {
+      const svc = getService(ctx, options)
+      const body = await readJson(req)
+      const stats = await svc.reindex({ force: !!body.force })
+      res.setHeader('Content-Type', 'application/json')
+      res.end(JSON.stringify(stats))
+    }, true)
+  )
+}