@soederpop/luca 0.0.23 → 0.0.25

package/src/introspection/generated.web.ts

@@ -1,7 +1,7 @@
 import { setBuildTimeData, setContainerBuildTimeData } from './index.js';
 
 // Auto-generated introspection registry data
-// Generated at: 2026-03-21T15:48:33.058Z
+// Generated at: 2026-03-22T06:53:17.219Z
 
 setBuildTimeData('features.containerLink', {
   "id": "features.containerLink",

package/src/node/container.ts

@@ -358,7 +358,7 @@ export class NodeContainer<
         return parse(path).dir
       },
       join(...paths: string[]) {
-        return join(cwd, ...paths);
+        return resolve(cwd, ...paths)
       },
       resolve(...paths: string[]) {
         return resolve(cwd, ...paths);

package/src/node/features/content-db.ts

@@ -3,8 +3,8 @@ import * as contentbaseExports from 'contentbase'
 import { parse, Collection, extractSections, type ModelDefinition } from 'contentbase'
 import { z } from 'zod'
 import { FeatureStateSchema, FeatureOptionsSchema, FeatureEventsSchema } from '../../schemas/base.js'
-import { join, dirname } from 'node:path'
-import { existsSync, readdirSync } from 'node:fs'
+import { realpathSync } from 'node:fs'
+import type { GrepOptions } from './grep.js'
 
 export const ContentDbStateSchema = FeatureStateSchema.extend({
   loaded: z.boolean().default(false).describe('Whether the content collection has been loaded and parsed'),
@@ -45,6 +45,71 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
   static override eventsSchema = ContentDbEventsSchema
   static { Feature.register(this, 'contentDb') }
 
+  /** Tools that any assistant can use to progressively explore this collection. */
+  static tools: Record<string, { schema: z.ZodType; handler?: Function }> = {
+    getCollectionOverview: {
+      schema: z.object({}).describe(
+        'Get a high-level overview of the document collection: models, document counts, directory tree, and search index status. Call this first to understand what is available.'
+      ),
+    },
+    listDocuments: {
+      schema: z.object({
+        model: z.string().optional().describe('Filter to documents belonging to this model name'),
+        glob: z.string().optional().describe('Glob pattern to filter document path IDs (e.g. "guides/*", "apis/**")'),
+      }).describe(
+        'List available document IDs in the collection, optionally filtered by model or glob pattern.'
+      ),
+    },
+    readDocument: {
+      schema: z.object({
+        id: z.string().describe('The document path ID to read (e.g. "guides/intro")'),
+        include: z.array(z.string()).optional().describe('Only return these section headings'),
+        exclude: z.array(z.string()).optional().describe('Remove these section headings from the output'),
+        meta: z.boolean().optional().describe('Include YAML frontmatter in the output'),
+      }).describe(
+        'Read a single document by its path ID. Use include/exclude to read only specific sections and avoid loading unnecessary content.'
+      ),
+    },
+    readMultipleDocuments: {
+      schema: z.object({
+        ids: z.array(z.string()).describe('Array of document path IDs to read'),
+        include: z.array(z.string()).optional().describe('Only return these section headings from each document'),
+        exclude: z.array(z.string()).optional().describe('Remove these section headings from each document'),
+        meta: z.boolean().optional().describe('Include YAML frontmatter in the output'),
+      }).describe(
+        'Read multiple documents at once, concatenated with dividers. Use include/exclude to focus on relevant sections.'
+      ),
+    },
+    queryDocuments: {
+      schema: z.object({
+        model: z.string().describe('The model name to query (e.g. "Plan", "Task")'),
+        where: z.string().optional().describe('Filter conditions as JSON string, e.g. \'{ "meta.status": "approved" }\' or \'{ "meta.priority": { "$gt": 3 } }\''),
+        sort: z.string().optional().describe('Sort specification as JSON string, e.g. \'{ "meta.priority": "desc" }\''),
+        limit: z.number().optional().describe('Maximum number of results to return'),
+        offset: z.number().optional().describe('Number of results to skip'),
+        select: z.array(z.string()).optional().describe('Fields to include in output (e.g. ["id", "title", "meta.status"])'),
+      }).describe(
+        'Query documents by model with MongoDB-style filtering, sorting, and pagination. Returns serialized model instances.'
+      ),
+    },
+    searchContent: {
+      schema: z.object({
+        pattern: z.string().describe('Regex pattern to search for across all documents'),
+        caseSensitive: z.boolean().optional().describe('Whether the search is case-sensitive (default: false)'),
+      }).describe(
+        'Text/regex search (grep) across all documents in the collection. Returns matching lines with file context.'
+      ),
+    },
+    semanticSearch: {
+      schema: z.object({
+        query: z.string().describe('Natural language search query'),
+        limit: z.number().optional().describe('Maximum number of results (default: 10)'),
+      }).describe(
+        'Semantic search across documents using keyword + vector similarity. Falls back to text search if no search index exists.'
+      ),
+    },
+  }
+
   override get initialState(): ContentDbState {
     return {
       ...super.initialState,
@@ -54,7 +119,7 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
 
   /** Whether the content database has been loaded. */
   get isLoaded() {
-    return this.state.get('loaded')
+    return this.state.get('started')
   }
 
   _collection?: Collection
@@ -82,7 +147,7 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
   /** Check if contentbase is resolvable via native import from the project root */
   private _canNativeImportContentbase(): boolean {
     const cwd = this.container.cwd
-    return existsSync(join(cwd, 'node_modules', 'contentbase'))
+    return this.container.fs.exists(this.container.paths.resolve(cwd, 'node_modules', 'contentbase'))
   }
 
   /** Seed the VM with virtual modules so models.ts can import from 'contentbase', 'zod', etc. */
@@ -122,6 +187,61 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
     return this.collection.modelDefinitions.map((d) => d.name)
   }
 
+  /**
+   * Returns the available document ids in the collection
+  */
+  get available() : string[] {
+	  return this.collection.available
+  }
+  
+  /**
+   * Render a tree view of the collection directory structure.
+   * Built with container.fs so it works without the `tree` binary.
+   */
+  renderTree(options?: { depth?: number; dirsOnly?: boolean }): string {
+    const maxDepth = options?.depth ?? Infinity
+    const dirsOnly = options?.dirsOnly ?? false
+    const fs = this.container.fs
+    const paths = this.container.paths
+    const root = this.collectionPath
+    const lines: string[] = [paths.basename(root)]
+
+    const walk = (dir: string, prefix: string, currentDepth: number) => {
+      if (currentDepth >= maxDepth) return
+      const entries: string[] = fs.readdirSync(dir).sort()
+      const filtered = entries.filter((e: string) => {
+        if (e.startsWith('.')) return false
+        if (dirsOnly) return fs.isDirectory(paths.resolve(dir, e))
+        return true
+      })
+
+      filtered.forEach((entry: string, i: number) => {
+        const isLast = i === filtered.length - 1
+        const connector = isLast ? '└── ' : '├── '
+        const fullPath = paths.resolve(dir, entry)
+        lines.push(`${prefix}${connector}${entry}`)
+        if (fs.isDirectory(fullPath)) {
+          walk(fullPath, prefix + (isLast ? '    ' : '│   '), currentDepth + 1)
+        }
+      })
+    }
+
+    walk(root, '', 0)
+    return lines.join('\n')
+  }
+
+  async grep(options: string | GrepOptions) {
+    if (typeof options === 'string') {
+      options = { pattern: options }
+    }
+    return this.container.feature('grep').search({
+      path: this.collectionPath,
+      include: ['**/*.md'],
+      exclude: ['.env', 'secrets'],
+      ...options,
+    })
+  }
+
   /**
    * Query documents belonging to a specific model definition.
    *
@@ -170,7 +290,7 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
     }
 
     await this.collection.load()
-    this.state.set('loaded', true)
+    this.state.set('started', true)
 
     return this
   }
@@ -315,7 +435,7 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
 
   /**
    * Lazily initialize the semanticSearch feature, attaching it to the container if needed.
-   * The dbPath defaults to `.contentbase/search.sqlite` relative to the collection root.
+   * The dbPath defaults to `~/.luca/contentbase/{hash}/search.sqlite` where hash is derived from the resolved collection path.
    */
   private async _getSemanticSearch(options?: { dbPath?: string; embeddingProvider?: string; embeddingModel?: string }) {
     if (this._semanticSearch?.state?.get('dbReady')) return this._semanticSearch
@@ -326,9 +446,10 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
       SemanticSearch.attach(this.container as any)
     }
 
-    // Put .contentbase at project root (dirname of docs/), not inside the docs folder
-    const projectRoot = dirname(this.collectionPath)
-    const dbPath = options?.dbPath ?? join(projectRoot, '.contentbase/search.sqlite')
+    // Store search index in ~/.luca/contentbase/{hash}/ keyed by the real (symlink-resolved) collection path
+    const realPath = realpathSync(this.collectionPath)
+    const pathHash = this.container.utils.hashObject(realPath).slice(0, 12)
+    const dbPath = options?.dbPath ?? this.container.paths.resolve(this.container.os.homedir, '.luca', 'contentbase', pathHash, 'search.sqlite')
     this._semanticSearch = (this.container as any).feature('semanticSearch', {
       dbPath,
       ...(options?.embeddingProvider ? { embeddingProvider: options.embeddingProvider } : {}),
@@ -343,10 +464,14 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
    * Check if a search index exists for this collection.
    */
   private _hasSearchIndex(): boolean {
-    const dbDir = join(dirname(this.collectionPath), '.contentbase')
-    if (!existsSync(dbDir)) return false
+    const realPath = realpathSync(this.collectionPath)
+    const pathHash = this.container.utils.hashObject(realPath).slice(0, 12)
+    const dbDir = this.container.paths.resolve(this.container.os.homedir, '.luca', 'contentbase', pathHash)
+
+    if (!this.container.fs.exists(dbDir)) return false
+
     try {
-      const files = readdirSync(dbDir) as string[]
+      const files = this.container.fs.readdirSync(dbDir)
       return files.some((f: string) => f.startsWith('search.') && f.endsWith('.sqlite'))
     } catch {
       return false
@@ -525,6 +650,119 @@ export class ContentDb extends Feature<ContentDbState, ContentDbOptions> {
     }
     return Object.fromEntries(queryChains)
   }
+  // ── Tool Methods ─────────────────────────────────────────────────
+  // These methods are auto-bound as tool handlers by toTools() because
+  // their names match the keys in static tools above.
+
+  /** Returns a high-level overview of the collection. */
+  async getCollectionOverview() {
+    if (!this.isLoaded) await this.load()
+
+    const modelCounts: Record<string, number> = {}
+    for (const def of this.collection.modelDefinitions) {
+      const count = await this.collection.query(def).count()
+      modelCounts[def.name] = count
+    }
+
+    return {
+      rootPath: this.collectionPath,
+      totalDocuments: this.available.length,
+      models: modelCounts,
+      tree: this.renderTree({ depth: 2 }),
+      hasSearchIndex: this._hasSearchIndex(),
+    }
+  }
+
+  /** List document IDs, optionally filtered by model or glob. */
+  async listDocuments(args: { model?: string; glob?: string }) {
+    if (!this.isLoaded) await this.load()
+
+    let ids = this.available
+
+    if (args.model) {
+      const def = this.models[args.model]
+      if (!def) return { error: `Unknown model "${args.model}". Available: ${this.modelNames.join(', ')}` }
+      const instances = await this.collection.query(def).fetchAll()
+      ids = instances.map((inst: any) => inst.id)
+    }
+
+    if (args.glob) {
+      const matched = this.collection.matchPaths(args.glob)
+      ids = ids.filter((id: string) => matched.includes(id))
+    }
+
+    return ids
+  }
+
+  /** Read a single document with optional section filtering. */
+  async readDocument(args: { id: string; include?: string[]; exclude?: string[]; meta?: boolean }) {
+    return this.read(args.id, args)
+  }
+
+  /** Read multiple documents with optional section filtering. */
+  async readMultipleDocuments(args: { ids: string[]; include?: string[]; exclude?: string[]; meta?: boolean }) {
+    return this.readMultiple(args.ids, args)
+  }
+
+  /** Query documents by model with filters, sort, limit. */
+  async queryDocuments(args: { model: string; where?: string; sort?: string; limit?: number; offset?: number; select?: string[] }) {
+    if (!this.isLoaded) await this.load()
+
+    const def = this.models[args.model]
+    if (!def) return { error: `Unknown model "${args.model}". Available: ${this.modelNames.join(', ')}` }
+
+    let q = this.collection.query(def)
+
+    if (args.where) {
+      const where: Record<string, any> = typeof args.where === 'string' ? JSON.parse(args.where) : args.where
+      for (const [path, value] of Object.entries(where)) {
+        q = q.where(path, value)
+      }
+    }
+    if (args.sort) {
+      const sort: Record<string, string> = typeof args.sort === 'string' ? JSON.parse(args.sort) : args.sort
+      for (const [path, dir] of Object.entries(sort)) {
+        q = q.sort(path, dir as 'asc' | 'desc')
+      }
+    }
+    if (args.limit) q = q.limit(args.limit)
+    if (args.offset) q = q.offset(args.offset)
+
+    const results = await q.fetchAll()
+
+    return results.map((inst: any) => {
+      const json = inst.toJSON()
+      if (args.select?.length) {
+        const picked: Record<string, any> = {}
+        for (const field of args.select) {
+          picked[field] = this.container.utils.lodash.get(json, field)
+        }
+        return picked
+      }
+      return json
+    })
+  }
+
+  /** Grep/text search across the collection. */
+  async searchContent(args: { pattern: string; caseSensitive?: boolean }) {
+    return this.grep({
+      pattern: args.pattern,
+      caseSensitive: args.caseSensitive ?? false,
+    } as GrepOptions)
+  }
+
+  /** Hybrid semantic search with graceful fallback to grep. */
+  async semanticSearch(args: { query: string; limit?: number }) {
+    try {
+      return await this.hybridSearch(args.query, { limit: args.limit ?? 10 })
+    } catch {
+      const grepResults = await this.grep({ pattern: args.query })
+      return {
+        results: grepResults,
+        note: 'No search index available — fell back to text search. Run `cbase embed` to enable semantic search.',
+      }
+    }
+  }
 }
 
-export default ContentDb
+export default ContentDb

package/src/node/features/git.ts

@@ -425,6 +425,96 @@ export class Git extends Feature {
         return output
     }
 
+    /**
+     * Extracts a folder (or entire repo) from a remote GitHub repository without cloning.
+     *
+     * Downloads the repo as a tarball and extracts only the specified subfolder,
+     * similar to how degit works. No .git history is included — just the files.
+     *
+     * Supports shorthand (`user/repo/path`), branch refs (`user/repo/path#branch`),
+     * and full GitHub URLs (`https://github.com/user/repo/tree/branch/path`).
+     *
+     * @param {object} options
+     * @param {string} options.source - Repository source in degit-style shorthand
+     * @param {string} options.destination - Local path to extract files into
+     * @param {string} [options.branch] - Branch, tag, or commit ref (overrides ref in source string)
+     * @returns {Promise<{ files: string[], source: { user: string, repo: string, ref: string, subdir: string } }>}
+     *
+     * @example
+     * ```typescript
+     * // Extract a subfolder
+     * await git.extractFolder({ source: 'soederpop/luca/src/assistants', destination: './my-assistants' })
+     *
+     * // Specific branch
+     * await git.extractFolder({ source: 'sveltejs/template', destination: './my-app', branch: 'main' })
+     *
+     * // Full GitHub URL
+     * await git.extractFolder({ source: 'https://github.com/user/repo/tree/main/examples', destination: './examples' })
+     * ```
+     */
+    async extractFolder({ source, destination, branch }: { source: string, destination: string, branch?: string }) {
+        const parsed = this._parseRemoteSource(source)
+        if (branch) parsed.ref = branch
+
+        const tarballUrl = `https://github.com/${parsed.user}/${parsed.repo}/archive/${parsed.ref}.tar.gz`
+        const stamp = Date.now()
+        const fs = this.container.feature('fs')
+        const proc = this.container.feature('proc')
+        const tmpBase = this.container.paths.resolve(this.container.feature('os').tmpdir, 'luca-degit')
+        fs.ensureFolder(tmpBase)
+        const tarPath = this.container.paths.resolve(tmpBase, `.degit-${stamp}.tar.gz`)
+        const tmpExtract = this.container.paths.resolve(tmpBase, `.degit-extract-${stamp}`)
+        const dest = destination.startsWith('/') ? destination : this.container.paths.resolve(destination)
+
+        const dl = this.container.feature('downloader')
+
+        await dl.download(tarballUrl, tarPath)
+
+        // Extract everything, strip the root archive directory (e.g. repo-commitsha/)
+        fs.ensureFolder(tmpExtract)
+        const extractResult = await proc.execAndCapture(`tar xzf ${tarPath} -C ${tmpExtract} --strip-components=1`)
+        if (extractResult.exitCode !== 0) {
+            await fs.rmdir(tmpExtract)
+            await fs.rm(tarPath)
+            throw new Error(`Failed to extract tarball: ${extractResult.stderr}`)
+        }
+
+        // Copy the subfolder (or everything) to destination
+        const sourceDir = parsed.subdir ? `${tmpExtract}/${parsed.subdir}` : tmpExtract
+        if (fs.existsSync(dest)) await fs.rmdir(dest)
+        fs.ensureFolder(dest)
+        fs.copy(sourceDir, dest, { overwrite: true })
+
+        // Cleanup temp files
+        await fs.rm(tarPath)
+        await fs.rmdir(tmpExtract)
+
+        const files = fs.readdirSync(dest)
+        return { files, source: parsed }
+    }
+
+    /** Parses degit-style source strings into components. */
+    private _parseRemoteSource(input: string) {
+        let ref = 'HEAD'
+        let str = input.replace(/^https?:\/\//, '')
+
+        // Handle github.com/user/repo/tree/branch/path URLs
+        const treeMatch = str.match(/^github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)(?:\/(.+))?$/)
+        if (treeMatch) {
+            return { user: treeMatch[1], repo: treeMatch[2], ref: treeMatch[3], subdir: treeMatch[4] || '' }
+        }
+
+        if (str.startsWith('github.com/')) str = str.replace('github.com/', '')
+        if (str.includes('#')) {
+            const parts = str.split('#')
+            str = parts[0]
+            ref = parts[1]
+        }
+
+        const parts = str.split('/')
+        return { user: parts[0], repo: parts[1], ref, subdir: parts.slice(2).join('/') }
+    }
+
     /**
      * Gets the commit history for a set of files or glob patterns.
      *

package/src/node/features/grep.ts

@@ -13,7 +13,7 @@ export type GrepMatch = {
     content: string
 }
 
-type GrepOptions = {
+export type GrepOptions = {
     /** Pattern to search for (string or regex) */
     pattern: string
     /** Directory or file to search in (defaults to container cwd) */

package/src/node/features/proc.ts

@@ -273,6 +273,7 @@ export class ChildProcess extends Feature {
   exec(command: string, options?: any): string {
     return execSync(command, {
       cwd: this.container.cwd,
+      stdio: ['pipe', 'pipe', 'pipe'],
       ...options,
     })
       .toString()

package/src/node/features/tts.ts

@@ -70,7 +70,7 @@ export class TTS extends Feature<TTSState, TTSOptions> {
 
   /** Directory where generated audio files are saved. */
   get outputDir(): string {
-    return this.options.outputDir || this.container.paths.join(this.container.feature('os').homedir, '.luca', 'tts-cache')
+    return this.options.outputDir || this.container.paths.resolve(this.container.feature('os').homedir, '.luca', 'tts-cache')
   }
 
   /** The 20 preset voice names available in Chatterbox Turbo. */

package/src/node/features/vm.ts

@@ -247,6 +247,54 @@ export class VM<
     return (await script.runInContext(context)) as T
   }
 
+  /**
+   * Execute code and capture all console output as structured JSON.
+   *
+   * Returns both the execution result and an array of every `console.*` call
+   * made during execution, each entry recording the method name and arguments.
+   *
+   * @param code - The JavaScript code to execute
+   * @param ctx - Context variables to make available to the executing code
+   * @returns The result, an array of captured console calls, and the context
+   *
+   * @example
+   * ```typescript
+   * const { result, console: calls } = await vm.runCaptured('console.log("hi"); console.warn("oh"); 42')
+   * // result === 42
+   * // calls === [{ method: 'log', args: ['hi'] }, { method: 'warn', args: ['oh'] }]
+   * ```
+   */
+  async runCaptured<T extends any>(code: string, ctx: any = {}): Promise<{
+    result: T
+    console: Array<{ method: string, args: any[] }>
+    context: vm.Context
+  }> {
+    const calls: Array<{ method: string, args: any[] }> = []
+    const captureConsole: Record<string, (...args: any[]) => void> = {}
+
+    for (const method of ['log', 'info', 'warn', 'error', 'debug', 'trace', 'dir', 'table'] as const) {
+      captureConsole[method] = (...args: any[]) => {
+        calls.push({ method, args })
+      }
+    }
+
+    const isExisting = this.isContext(ctx)
+    const context = isExisting ? ctx : this.createContext({ ...ctx, console: captureConsole })
+
+    // For existing contexts, swap console in for the run and restore after
+    const prevConsole = isExisting ? ctx.console : undefined
+    if (isExisting) ctx.console = captureConsole
+
+    const wrapped = this.wrapTopLevelAwait(code)
+    const script = this.createScript(wrapped)
+    try {
+      const result = (await script.runInContext(context)) as T
+      return { result, console: calls, context }
+    } finally {
+      if (isExisting) ctx.console = prevConsole
+    }
+  }
+
   /**
    * Execute JavaScript code synchronously in a controlled environment.
    *

package/src/scaffolds/generated.ts

@@ -1,5 +1,5 @@
 // Auto-generated scaffold and MCP readme content
-// Generated at: 2026-03-21T05:25:03.287Z
+// Generated at: 2026-03-22T06:53:18.105Z
 // Source: docs/scaffolds/*.md, docs/examples/assistant/, and docs/mcp/readme.md
 //
 // Do not edit manually. Run: luca build-scaffolds
@@ -1669,7 +1669,7 @@ import { Server, servers, ServerStateSchema } from '@soederpop/luca'
 import { commands, CommandOptionsSchema } from '@soederpop/luca'
 \`\`\`
 
-Never import from \`fs\`, \`path\`, \`crypto\`, or other Node builtins. Never import third-party packages in consumer code. The only exception is inside helper implementations themselves — a feature that wraps a library may import it.
+Never import from \`fs\`, \`path\`, \`crypto\`, or other Node builtins. Never import third-party packages in consumer code. If a container feature wraps the functionality, use it.
 
 ## Zod v4
 

package/assistants/architect/CORE.md

@@ -1,3 +0,0 @@
-# Luca Framework Assistant 
-
-Your job is to help people use the Luca framework

package/assistants/architect/hooks.ts

@@ -1,3 +0,0 @@
-export function started() {
-	console.log('Assistant started!')
-}

package/assistants/architect/tools.ts

@@ -1,10 +0,0 @@
-import { z } from 'zod'
-
-export const schemas = {
-	README: z.object({}).describe('CALL THIS README FUNCTION AS EARLY AS POSSIBLE')
-}
-
-export function README(options: z.infer<typeof schemas.README>) {
-	return 'YO YO'
-}
-