@soederpop/luca 0.0.2

package/docs/tutorials/18-semantic-search.md

@@ -0,0 +1,334 @@
+---
+title: Semantic Search
+tags: [semantic-search, embeddings, vector-search, bm25, hybrid-search, sqlite, contentdb]
+---
+
+# Semantic Search
+
+Luca's `semanticSearch` feature provides BM25 keyword search, vector similarity search, and hybrid search with Reciprocal Rank Fusion -- all backed by SQLite. It chunks documents intelligently, generates embeddings via OpenAI or a local GGUF model, and stores everything in a single `.sqlite` file.
+
+## Quick Start with ContentDb
+
+The fastest way to use semantic search is through the `contentDb` feature, which handles indexing and querying automatically:
+
+```typescript
+import container from '@soederpop/luca'
+
+const db = container.feature('contentDb', { rootPath: './docs' })
+await db.load()
+
+// Build the search index (generates embeddings for all documents)
+await db.buildSearchIndex({
+  onProgress: (indexed, total) => console.log(`${indexed}/${total}`)
+})
+
+// Search your documents
+const results = await db.hybridSearch('how does authentication work')
+for (const r of results) {
+  console.log(`${r.title} (score: ${r.score.toFixed(3)})`)
+  console.log(`  ${r.snippet}`)
+}
+```
+
+ContentDb provides three search methods that delegate to the underlying semanticSearch feature:
+
+```typescript
+// BM25 keyword search -- best for exact term matching
+await db.search('OAuth2 token refresh')
+
+// Vector similarity search -- finds conceptually related documents
+await db.vectorSearch('how do users log in')
+
+// Hybrid search -- combines both via Reciprocal Rank Fusion (recommended)
+await db.hybridSearch('authentication flow', { limit: 5 })
+```
+
+## Using SemanticSearch Directly
+
+For more control, use the `semanticSearch` feature directly:
+
+```typescript
+import container from '@soederpop/luca'
+import { SemanticSearch } from '@soederpop/luca/node/features/semantic-search'
+
+// Attach the feature to the container
+SemanticSearch.attach(container)
+
+const search = container.feature('semanticSearch', {
+  dbPath: '.contentbase/search.sqlite',
+  embeddingProvider: 'openai',        // or 'local'
+  embeddingModel: 'text-embedding-3-small',
+  chunkStrategy: 'section',           // 'section' | 'fixed' | 'document'
+  chunkSize: 900,
+})
+
+await search.initDb()
+```
+
+## Indexing Documents
+
+Documents are represented as `DocumentInput` objects with optional section metadata:
+
+```typescript
+await search.indexDocuments([
+  {
+    pathId: 'guides/auth',
+    model: 'Guide',
+    title: 'Authentication Guide',
+    meta: { status: 'published', category: 'security' },
+    content: 'Full document content here...',
+    sections: [
+      {
+        heading: 'OAuth2 Flow',
+        headingPath: 'Authentication Guide > OAuth2 Flow',
+        content: 'OAuth2 uses authorization codes and tokens...',
+        level: 2,
+      },
+      {
+        heading: 'Session Management',
+        headingPath: 'Authentication Guide > Session Management',
+        content: 'Sessions are stored server-side with a cookie...',
+        level: 2,
+      },
+    ],
+  },
+  {
+    pathId: 'guides/deployment',
+    title: 'Deployment Guide',
+    content: 'How to deploy your application...',
+  },
+])
+```
+
+The `indexDocuments` method:
+1. Stores documents in SQLite with FTS5 full-text indexing
+2. Chunks each document based on the configured strategy
+3. Generates embeddings for every chunk
+4. Stores embeddings as BLOBs alongside the chunk text
+
+## Chunking Strategies
+
+The feature splits documents into chunks before embedding. Choose a strategy based on your content:
+
+### Section (default)
+
+Splits at heading boundaries (`## H2`, `### H3`). Each section becomes a chunk, prefixed with the heading path for context. Falls back to fixed chunking if the document has no sections.
+
+```typescript
+const search = container.feature('semanticSearch', {
+  chunkStrategy: 'section',
+  chunkSize: 900,  // max tokens per chunk (sections exceeding this are split at paragraphs)
+})
+```
+
+Best for: structured documents with clear heading hierarchies.
+
+### Fixed
+
+Splits by word count with configurable overlap between chunks:
+
+```typescript
+const search = container.feature('semanticSearch', {
+  chunkStrategy: 'fixed',
+  chunkSize: 900,
+  chunkOverlap: 0.15,  // 15% overlap between adjacent chunks
+})
+```
+
+Best for: unstructured prose, logs, or transcripts.
+
+### Document
+
+One chunk per document -- no splitting:
+
+```typescript
+const search = container.feature('semanticSearch', {
+  chunkStrategy: 'document',
+})
+```
+
+Best for: short documents where splitting would lose context.
+
+## Search Methods
+
+### BM25 Keyword Search
+
+Uses SQLite FTS5 with Porter stemming for traditional keyword matching:
+
+```typescript
+const results = await search.search('authentication tokens', {
+  limit: 10,
+  model: 'Guide',                        // filter by document model
+  where: { status: 'published' },         // filter by metadata fields
+})
+```
+
+Returns results ranked by BM25 relevance with highlighted snippets.
+
+### Vector Similarity Search
+
+Embeds the query and computes cosine similarity against all stored chunk embeddings:
+
+```typescript
+const results = await search.vectorSearch('how do users prove their identity', {
+  limit: 10,
+})
+```
+
+Finds conceptually related content even without keyword overlap. Results are deduplicated by document, keeping the best-scoring chunk per document.
+
+### Hybrid Search (Recommended)
+
+Runs both BM25 and vector search in parallel, then fuses results using Reciprocal Rank Fusion:
+
+```typescript
+const results = await search.hybridSearch('authentication flow', {
+  limit: 10,
+  model: 'Guide',
+  where: { category: 'security' },
+})
+```
+
+This gives the best results for most queries -- keyword precision combined with semantic recall.
+
+## Search Results
+
+All search methods return `SearchResult[]`:
+
+```typescript
+interface SearchResult {
+  pathId: string          // document identifier
+  model: string           // content model name
+  title: string           // document title
+  meta: Record<string, any>  // document metadata
+  score: number           // relevance score
+  snippet: string         // matched text excerpt
+  matchedSection?: string // section heading where the match occurred
+  headingPath?: string    // full heading breadcrumb (e.g. "Auth > OAuth2 > Tokens")
+}
+```
+
+## Embedding Providers
+
+### OpenAI (default)
+
+Uses the OpenAI embeddings API. Requires an `openai` client registered in the container.
+
+```typescript
+const search = container.feature('semanticSearch', {
+  embeddingProvider: 'openai',
+  embeddingModel: 'text-embedding-3-small',  // 1536 dimensions
+  // also available: 'text-embedding-3-large' (3072 dimensions)
+})
+```
+
+### Local (GGUF)
+
+Runs embeddings locally using `node-llama-cpp` with a GGUF model file:
+
+```typescript
+const search = container.feature('semanticSearch', {
+  embeddingProvider: 'local',
+  embeddingModel: 'embedding-gemma-300M-Q8_0',  // 768 dimensions
+})
+
+// Install the dependency if needed
+await search.installLocalEmbeddings(process.cwd())
+```
+
+Local models are loaded from `~/.cache/luca/models/` or `~/.cache/qmd/models/`. The model is kept in memory and automatically disposed after 5 minutes of inactivity.
+
+## Index Management
+
+### Incremental Updates
+
+The feature tracks content hashes to avoid re-embedding unchanged documents:
+
+```typescript
+// Check if a document needs re-indexing
+if (search.needsReindex(doc)) {
+  search.removeDocument(doc.pathId)
+  await search.indexDocuments([doc])
+}
+```
+
+### Remove Stale Documents
+
+Clean up documents that no longer exist in your collection:
+
+```typescript
+const currentIds = ['guides/auth', 'guides/deployment']
+search.removeStale(currentIds)  // deletes any indexed docs not in this list
+```
+
+### Full Reindex
+
+Clear everything and start fresh:
+
+```typescript
+await search.reindex()  // clears all data
+await search.indexDocuments(allDocs)  // re-index everything
+```
+
+### Index Status
+
+```typescript
+const stats = search.getStats()
+// {
+//   documentCount: 42,
+//   chunkCount: 187,
+//   embeddingCount: 187,
+//   lastIndexedAt: '2026-03-06T...',
+//   provider: 'openai',
+//   model: 'text-embedding-3-small',
+//   dimensions: 1536,
+//   dbSizeBytes: 2457600,
+// }
+```
+
+## Database Scoping
+
+Each provider/model combination gets its own SQLite file. If you configure `dbPath: '.contentbase/search.sqlite'` with the OpenAI provider and `text-embedding-3-small` model, the actual file will be `.contentbase/search.openai-text-embedding-3-small.sqlite`. This prevents dimension mismatches if you switch providers.
+
+## ContentDb Integration Details
+
+When using `contentDb.buildSearchIndex()`, the feature automatically:
+
+- Extracts sections from your markdown documents at H2 boundaries
+- Converts each document to a `DocumentInput` with pathId, title, meta, and sections
+- Skips unchanged documents (incremental by default)
+- Removes documents that no longer exist in the collection
+
+```typescript
+const db = container.feature('contentDb', { rootPath: './docs' })
+await db.load()
+
+// Incremental update (default)
+const { indexed, total } = await db.buildSearchIndex()
+console.log(`Indexed ${indexed} of ${total} documents`)
+
+// Force full rebuild
+await db.rebuildSearchIndex()
+
+// Check index health
+console.log(db.searchIndexStatus)
+```
+
+## Lifecycle
+
+Always close the feature when done to release the SQLite connection and any loaded models:
+
+```typescript
+await search.close()
+```
+
+The feature emits events you can listen to:
+
+```typescript
+search.on('dbReady', () => console.log('Database initialized'))
+search.on('indexed', ({ documents, chunks }) => {
+  console.log(`Indexed ${documents} docs (${chunks} chunks)`)
+})
+search.on('modelLoaded', () => console.log('Local embedding model loaded'))
+search.on('modelDisposed', () => console.log('Local embedding model released'))
+```

package/index.ts

@@ -0,0 +1 @@
+console.log("Hello via Bun!");

package/luca.console.ts

@@ -0,0 +1,9 @@
+import type { AGIContainer } from '@soederpop/luca/agi'
+
+declare global {
+	var container: AGIContainer 
+}
+
+export const ass = container.feature('assistant', {
+	folder: 'assistants/project-owner',
+})

package/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from luca2!")
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -0,0 +1,154 @@
+{
+  "name": "@soederpop/luca",
+  "version": "0.0.2",
+  "private": false,
+  "description": "lightweight universal conversational architecture",
+  "author": "jon soeder <jon@soederpop.com>",
+  "type": "module",
+  "repository": "https://github.com/soederpop/luca",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "conversational",
+    "architecture"
+  ],
+  "main": "./src/node.ts",
+  "types": "./src/node.ts",
+  "browser": "./src/browser.ts",
+  "exports": {
+    ".": "./src/node.ts",
+    "./node": "./src/node.ts",
+    "./node/*": "./src/node/*",
+    "./web": "./src/browser.ts",
+    "./web/*": "./src/web/*",
+    "./agi": "./src/agi/index.ts",
+    "./agi/*": "./src/agi/*",
+    "./schemas": "./src/schemas/base.ts",
+    "./schemas/*": "./src/schemas/*",
+    "./container": "./src/container.ts",
+    "./client": "./src/client.ts",
+    "./clients/*": "./src/clients/*",
+    "./feature": "./src/feature.ts",
+    "./react": "./src/react/index.ts"
+  },
+  "bin": {
+    "luca": "./src/cli/cli.ts"
+  },
+  "scripts": {
+    "serve": "bun build:web && luca serve --any-port",
+    "console": "ts-node --esm scripts/console.ts",
+    "lint": "eslint src/ --ext .js,.jsx,.ts,.tsx",
+    "clean": "rm -rf dist build package",
+    "build:web": "bun run scripts/build-web.ts",
+    "build:introspection": "bun run src/cli/cli.ts update-introspection",
+    "compile": "bun build ./src/cli/cli.ts --compile --outfile dist/luca --external node-llama-cpp",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "build:scaffolds": "bun run src/cli/cli.ts build-scaffolds",
+    "test": "bun test test/*.test.ts",
+    "update-all-docs": "bun run test && bun run build:introspection && bun run src/cli/cli.ts generate-api-docs && bun run build:scaffolds",
+    "precommit": "bun run update-all-docs && git add docs/apis/ src/introspection/generated.*.ts src/scaffolds/generated.ts && bun compile",
+    "test:integration": "bun test ./test-integration/"
+  },
+  "devDependencies": {
+    "@types/cacache": "^15.0.1",
+    "@types/chalk": "^2.2.0",
+    "@types/child-process-promise": "^2.2.2",
+    "@types/chokidar": "^2.1.3",
+    "@types/cors": "^2.8.13",
+    "@types/detect-port": "^1.3.2",
+    "@types/dotenv": "^8.2.0",
+    "@types/express": "^4.17.17",
+    "@types/figlet": "^1.5.5",
+    "@types/glob": "^8.1.0",
+    "@types/inquirer": "^9.0.3",
+    "@types/jest": "^27.4.1",
+    "@types/js-yaml": "^4.0.5",
+    "@types/lodash-es": "^4.17.7",
+    "@types/micromatch": "^4.0.2",
+    "@types/minimist": "^1.2.2",
+    "@types/node": "^18.15.11",
+    "@types/node-uuid": "^0.0.29",
+    "@types/object-hash": "^3.0.2",
+    "@types/react": "^18.2.0",
+    "@types/uuid": "^9.0.1",
+    "@types/ws": "^8.5.4",
+    "@typescript-eslint/eslint-plugin": "^5.20.0",
+    "@typescript-eslint/parser": "^5.20.0",
+    "eslint": "^8.14.0",
+    "mkdist": "^1.3.0",
+    "prettier": "^2.8.7",
+    "ts-node": "^11.0.0-beta.1"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.1",
+    "@ngrok/ngrok": "^1.5.1",
+    "@openai/codex": "^0.99.0",
+    "@resvg/resvg-js": "^2.6.2",
+    "@supabase/supabase-js": "^2.95.3",
+    "@types/marked": "^6.0.0",
+    "@types/marked-terminal": "^6.1.1",
+    "axios": "^1.3.5",
+    "cacache": "^17.0.7",
+    "chalk": "^5.2.0",
+    "child-process-promise": "^2.2.1",
+    "chokidar": "^3.5.3",
+    "cli-markdown": "^3.5.0",
+    "compromise": "^14.14.5",
+    "contentbase": "",
+    "cors": "^2.8.5",
+    "cross-fetch": "^4.1.0",
+    "detect-port": "^1.5.1",
+    "dotenv": "^17.2.4",
+    "endent": "^2.1.0",
+    "esbuild-wasm": "0.17.18",
+    "excalidraw-to-svg": "^3.1.0",
+    "express": "^4.18.2",
+    "figlet": "^1.6.0",
+    "glob": "^11.0.2",
+    "googleapis": "^171.4.0",
+    "grammy": "^1.40.0",
+    "inflect": "^0.5.0",
+    "ink": "^6.7.0",
+    "inquirer": "^9.1.5",
+    "isomorphic-vm": "^0.0.1",
+    "isomorphic-ws": "^5.0.0",
+    "js-tiktoken": "^1.0.21",
+    "js-yaml": "^4.1.0",
+    "lodash-es": "^4.17.21",
+    "marked": "^15.0.12",
+    "marked-terminal": "^7.3.0",
+    "mdast-util-to-markdown": "^1.5.0",
+    "mdast-util-to-string": "^3.2.0",
+    "micromatch": "^4.0.5",
+    "minimist": "^1.2.8",
+    "node-uuid": "^1.4.8",
+    "object-hash": "^3.0.0",
+    "openai": "^5.1.1",
+    "opener": "^1.5.2",
+    "react": "^19.2.4",
+    "react-devtools-core": "^7.0.1",
+    "react-dom": "^19.2.4",
+    "react-reconciler": "^0.33.0",
+    "remark-gfm": "^3.0.1",
+    "rimraf": "^5.0.0",
+    "typescript": "^5.9.3",
+    "unist-util-find-after": "^4.0.1",
+    "unist-util-find-all-after": "^4.0.1",
+    "unist-util-find-all-before": "^4.0.1",
+    "unist-util-find-before": "^3.0.1",
+    "unist-util-select": "^4.0.3",
+    "unist-util-visit": "^4.1.2",
+    "wink-eng-lite-web-model": "^1.8.1",
+    "wink-nlp": "^2.4.0",
+    "ws": "^8.13.0",
+    "zod": "^4.0.0"
+  },
+  "optionalDependencies": {
+    "node-llama-cpp": "^3.17.1"
+  },
+  "luca": {
+    "aliases": ["luca", "framework", "container", "runtime"]
+  }
+}

package/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "luca2"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = []

package/scripts/animations/chrome-glitch.ts

@@ -0,0 +1,55 @@
+import type { AsciiAnimation } from './types'
+
+const frames = [
+  String.raw`
+      ######   ######  ########
+      ##  ##   ##  ##     ##
+      ######   ##  ##     ##
+      ##  ##   ##  ##     ##
+      ##  ##   ######     ##
+      ##  ##   ######     ##
+
+         [o_o]  channel-open
+          /|\   packet-sync
+          / \   pulse: 71%
+
+      <<..................>>
+`,
+  String.raw`
+      ######   ######  ########
+      ##  ##   ##  ##     ##
+      ######   ##  ##     ##
+      ##  ##   ##  ##     ##
+      ##  ##   ######     ##
+      ##  ##   ######     ##
+
+         [0_0]  channel-open
+          /|\   packet-sync
+          / \   pulse: 84%
+
+      <<##..##..##..##..##..>>
+`,
+  String.raw`
+      ######   ######  ########
+      ##  ##   ##  ##     ##
+      ######   ##  ##     ##
+      ##  ##   ##  ##     ##
+      ##  ##   ######     ##
+      ##  ##   ######     ##
+
+         [O_O]  channel-open
+          /|\   packet-sync
+          / \   pulse: 96%
+
+      <<##################>>
+`,
+]
+
+const chromeGlitch: AsciiAnimation = {
+  id: 'chromeGlitch',
+  name: 'Chrome Glitch Totem',
+  fps: 5,
+  frames,
+}
+
+export default chromeGlitch

package/scripts/animations/index.ts

@@ -0,0 +1,16 @@
+import type { AsciiAnimation } from './types'
+import neonPulse from './neon-pulse'
+import chromeGlitch from './chrome-glitch'
+
+const animations: AsciiAnimation[] = [neonPulse, chromeGlitch]
+
+export function listAnimations(): AsciiAnimation[] {
+  return animations
+}
+
+export function resolveAnimation(id: string): AsciiAnimation {
+  const match = animations.find((animation) => animation.id.toLowerCase() === id.toLowerCase())
+  if (match) return match
+  if (!animations[0]) throw new Error('No animations are registered.')
+  return animations[0]
+}

package/scripts/animations/neon-pulse.ts

@@ -0,0 +1,64 @@
+import type { AsciiAnimation } from './types'
+
+const frames = [
+  String.raw`
+             .-""-.
+           .' .--. '.
+          /  /    \  \
+          | | 0  0 | |
+          | |  __  | |
+          | | /__\ | |
+          | | \__/ | |
+          | |      | |
+         /|  '.__.'  |\
+       .' |   /::\   | '.
+      /   |  |::::|  |   \
+     /____|__|::::|__|____\
+        /_/   |::|   \_\
+             /____\
+     ~ ~ ~  SIGNAL: STABLE  ~ ~ ~
+`,
+  String.raw`
+             .-""-.
+           .' .--. '.
+          /  / /\ \  \
+          | | ^  ^ | |
+          | |  --  | |
+          | | /__\ | |
+          | | \__/ | |
+          | |  /\  | |
+         /|  '.__.'  |\
+       .' |  _/::\_  | '.
+      /   | |::::::| |   \
+     /____|_|::::::|_|____\
+        /_/   |::|   \_\
+             /____\
+     ~ ~ ~  SIGNAL: BREATHING  ~ ~ ~
+`,
+  String.raw`
+             .-""-.
+           .' .--. '.
+          /  / __ \  \
+          | | o  o | |
+          | |  ..  | |
+          | | /__\ | |
+          | | \__/ | |
+          | | .--. | |
+         /|  '.__.'  |\
+       .' |   /::\   | '.
+      /   |  |::::|  |   \
+     /____|__|::::|__|____\
+        /_/   |::|   \_\
+             /____\
+     ~ ~ ~  SIGNAL: SCANNING  ~ ~ ~
+`,
+]
+
+const neonPulse: AsciiAnimation = {
+  id: 'neonPulse',
+  name: 'Neon Pulse Bot',
+  fps: 4,
+  frames,
+}
+
+export default neonPulse

package/scripts/animations/types.ts

@@ -0,0 +1,6 @@
+export interface AsciiAnimation {
+  id: string
+  name: string
+  fps: number
+  frames: string[]
+}

package/scripts/build-web.ts

@@ -0,0 +1,28 @@
+// @ts-nocheck
+import path from "path"
+
+const result = await Bun.build({
+  entrypoints: [path.resolve(import.meta.dir, "../src/browser.ts")],
+  outdir: path.resolve(import.meta.dir, "../dist"),
+  target: "browser",
+  format: "esm",
+  naming: "browser.js",
+  external: [
+    "react",
+    "react-dom",
+  ],
+  define: {
+    "process.env.NODE_ENV": JSON.stringify("production"),
+    "process.env.NODE_DEBUG": JSON.stringify(""),
+  },
+})
+
+if (!result.success) {
+  console.error("Build failed:")
+  for (const log of result.logs) {
+    console.error(log)
+  }
+  process.exit(1)
+}
+
+console.log(`Built browser bundle: dist/browser.js (${(result.outputs[0].size / 1024).toFixed(1)}KB)`)