@cdoing/core 0.1.0

package/src/index.ts

@@ -0,0 +1,114 @@
+// Tools
+export { ToolRegistry } from "./tools/registry";
+export { FileReadTool } from "./tools/file-read";
+export { FileWriteTool } from "./tools/file-write";
+export { FileEditTool } from "./tools/file-edit";
+export { GlobSearchTool } from "./tools/glob-search";
+export { GrepSearchTool } from "./tools/grep-search";
+export { ShellExecTool } from "./tools/shell-exec";
+export { FileRunTool } from "./tools/file-run";
+export { CodeVerifyTool } from "./tools/code-verify";
+export { WebFetchTool } from "./tools/web-fetch";
+export { WebSearchTool } from "./tools/web-search";
+export { SubAgentTool } from "./tools/sub-agent";
+export type { SubAgentRunnerFactory } from "./tools/sub-agent";
+export { SubAgentManager } from "./tools/sub-agent-manager";
+export type { SubAgentStatus, SubAgentEntry } from "./tools/sub-agent-manager";
+export { SubAgentStatusTool } from "./tools/sub-agent-status";
+export { SubAgentTerminateTool } from "./tools/sub-agent-terminate";
+export { TodoTool } from "./tools/todo";
+export { SystemInfoTool } from "./tools/system-info";
+export { MultiEditTool } from "./tools/multi-edit";
+
+export { ListDirTool } from "./tools/list-dir";
+export { ViewDiffTool } from "./tools/view-diff";
+export { ViewRepoMapTool } from "./tools/view-repo-map";
+export { CodebaseSearchTool } from "./tools/codebase-search";
+export { ASTEditTool } from "./tools/ast-edit";
+export { NotebookEditTool } from "./tools/notebook-edit";
+export type { ToolDefinition, ToolResult, BaseTool } from "./tools/types";
+
+// Search matching utilities
+export { findSearchMatch, findAllSearchMatches, executeFindAndReplace, executeMultiFindAndReplace, isUnifiedDiff, applyUnifiedDiff } from "./utils/search-match";
+
+// Streaming diff utilities
+export { streamDeterministicDiff, streamUnifiedDiff, StreamingDiffAccumulator } from "./utils/streaming-diff";
+export type { DiffChunk, DiffChunkCallback } from "./utils/streaming-diff";
+
+// Lazy apply — placeholder expansion for LLM-generated edits
+export { hasPlaceholders, expandPlaceholders, isPlaceholderLine } from "./utils/lazy-apply";
+export type { LazyApplyResult } from "./utils/lazy-apply";
+
+// Permissions
+export { PermissionManager, PermissionMode } from "./permissions";
+export type { PermissionRule, PermissionScope, PermissionPromptFn } from "./permissions";
+
+// Sandbox
+export { SandboxManager } from "./sandbox";
+export { defaultSandboxConfig } from "./sandbox";
+export type { SandboxConfig, SandboxMode, SandboxCheckResult, SandboxFilesystemConfig, SandboxNetworkConfig } from "./sandbox/types";
+
+// Hooks
+export { HookManager } from "./hooks";
+export type { HookDefinition, HookResult } from "./hooks";
+
+// Utilities
+export { safePath } from "./utils/path-safety";
+export { loadIgnorePatterns } from "./utils/gitignore";
+export { loadProjectConfig, getProjectConfigPath } from "./utils/project-config";
+export { MemoryStore } from "./utils/memory";
+export type { MemoryEntry } from "./utils/memory";
+export { TodoStore } from "./utils/todo";
+export type { TodoItem, TodoStatus } from "./utils/todo";
+
+// Context Providers — pluggable @ mention system
+export { ContextProviderRegistry } from "./context-providers/registry";
+export type { ContextProvider, ContextResult, ContextResolveOptions } from "./context-providers/types";
+export { TerminalContextProvider } from "./context-providers/terminal";
+export { OpenFilesContextProvider } from "./context-providers/open-files";
+export { UrlContextProvider } from "./context-providers/url";
+export { TreeContextProvider } from "./context-providers/tree";
+export { ProblemsContextProvider } from "./context-providers/problems";
+export { CodebaseContextProvider } from "./context-providers/codebase";
+export { ClipboardContextProvider } from "./context-providers/clipboard";
+export { FileIncludeContextProvider } from "./context-providers/file-include";
+export { GitContextProvider } from "./context-providers/git";
+export { DiffContextProvider } from "./context-providers/diff";
+export { FolderContextProvider } from "./context-providers/folder";
+export { DocsContextProvider } from "./context-providers/docs";
+
+// Project Rules — hierarchical glob-scoped rules
+export { RulesManager } from "./rules/manager";
+export type { Rule, RuleSource } from "./rules/types";
+
+// Plan Mode — read-only planning before execution
+export { PlanManager } from "./plan/manager";
+export type { Plan, PlanStep, PlanStatus } from "./plan/manager";
+
+// MCP Server Support — Model Context Protocol
+export { McpManager } from "./mcp/manager";
+export type { McpServerConfig, McpTool } from "./mcp/manager";
+
+// Effort Level Control — adjusts analysis depth
+export { EffortManager } from "./effort";
+export type { EffortLevel, EffortConfig } from "./effort";
+
+// OAuth — shared credential storage, PKCE, token management
+export {
+  saveOAuthTokens,
+  loadOAuthTokens,
+  clearOAuthTokens,
+  isOAuthExpired,
+  refreshAccessToken,
+  resolveOAuthToken,
+  generateOAuthUrl,
+  exchangeOAuthCode,
+  getOAuthStatus,
+} from "./oauth";
+export type { OAuthTokens } from "./oauth";
+
+// Codebase Indexing — FTS5 + embeddings
+export { CodebaseIndexer, IndexDatabase, chunkDocument, RecentEditsCache } from "./indexing";
+export type { CachedEdit } from "./indexing";
+export type { EmbeddingProvider } from "./indexing";
+export type { SearchResult, IndexingProgress, IndexStats, ChunkWithMeta } from "./indexing";

package/src/indexing/README.md

@@ -0,0 +1,267 @@
+# Codebase Indexing System
+
+## Overview
+
+The indexing system provides fast, ranked code search across the entire codebase using **SQLite FTS5** (full-text search with BM25 ranking) and optional **vector embeddings** (cosine similarity). It powers both the `codebase_search` tool and the `@codebase` context provider.
+
+## Architecture
+
+```
+┌──────────────────────────────────────────────────┐
+│                 CodebaseIndexer                    │
+│                                                    │
+│  1. Scan files (respects .gitignore)               │
+│  2. SHA-256 cache key per file                     │
+│  3. Diff against catalog (incremental updates)     │
+│  4. Chunk files (code-aware / basic / markdown)    │
+│  5. Insert into SQLite                             │
+│  6. Build FTS5 index (trigram tokenizer)            │
+│  7. Optional: compute embeddings                   │
+│                                                    │
+│  ┌──────────┐  ┌───────────┐  ┌──────────────┐   │
+│  │  Chunks   │  │  FTS5     │  │  Embeddings  │   │
+│  │  (SQLite) │  │  (BM25)   │  │  (Vectors)   │   │
+│  └──────────┘  └───────────┘  └──────────────┘   │
+│       │              │               │             │
+│       └──────────────┴───────────────┘             │
+│                      │                             │
+│              search(query)                         │
+│           FTS(35%) + Embeddings(65%)                │
+│              → deduplicated results                │
+└──────────────────────────────────────────────────┘
+```
+
+## How It Works
+
+### 1. File Scanning
+
+- Recursively scans the working directory
+- Respects `.gitignore` patterns
+- Skips: `node_modules`, `.git`, `dist`, `build`, `__pycache__`, `coverage`, etc.
+- Skips files > 1MB
+- Supports 40+ file extensions (code, config, docs)
+
+### 2. Incremental Updates
+
+Files are tracked in a `index_catalog` table with SHA-256 content hashes:
+
+```
+index_catalog:
+  path, cacheKey (SHA-256), lastUpdated, directory
+```
+
+On each index run:
+- **New files** → chunk + index
+- **Modified files** (hash changed) → delete old chunks, re-index
+- **Deleted files** → remove from all tables
+- **Unchanged files** → skip entirely
+
+This makes re-indexing fast — only changed files are processed.
+
+### 3. Chunking Strategies
+
+Three strategies based on file type:
+
+#### Code-Aware Chunking (`.ts`, `.js`, `.py`, `.go`, `.rs`, etc.)
+Splits on function/class boundaries using regex patterns:
+- Function declarations, arrow functions, class definitions
+- Method declarations, interfaces, type aliases, enums
+- Python `def`/`class`, Go `func`, Rust `fn`/`struct`/`impl`
+
+Falls back to line-based splitting if a chunk exceeds size limits.
+
+#### Markdown Chunking (`.md`, `.mdx`)
+Splits on header boundaries (`# H1`, `## H2`, `### H3`), keeping each section as a chunk.
+
+#### Basic Chunking (everything else)
+Line-based splitting with ~17 lines per chunk (targeting ~384 tokens) and 3-line overlap between chunks.
+
+All strategies:
+- Target ~384 tokens per chunk (~1400 chars)
+- Merge tiny chunks (< 100 chars) with neighbors
+- Track `startLine` and `endLine` for source mapping
+
+### 4. SQLite FTS5 Full-Text Search
+
+Uses SQLite's FTS5 extension with **trigram tokenizer** for substring matching:
+
+```sql
+CREATE VIRTUAL TABLE fts USING fts5(
+  path,
+  content,
+  tokenize = 'trigram'
+);
+```
+
+**How trigram works:** Text is broken into overlapping 3-character sequences. "function" → "fun", "unc", "nct", "cti", "tio", "ion". This enables substring matching without word boundaries.
+
+**BM25 ranking** with 10x path boost:
+```sql
+ORDER BY bm25(fts, 10.0)
+```
+Matches in file paths (e.g., searching "auth" matches `src/auth/middleware.ts`) rank 10x higher than matches in content only.
+
+### 5. Vector Embeddings (Optional)
+
+When an `EmbeddingProvider` is configured:
+
+1. Each chunk's text is sent to the embedding model (batched)
+2. Vectors stored in SQLite as JSON arrays
+3. Search uses **cosine similarity** computed in-process:
+
+```typescript
+cosine_similarity(query_vector, chunk_vector) = dot(a,b) / (|a| * |b|)
+```
+
+No external vector database needed — all stored in the same SQLite file.
+
+### 6. Combined Search Pipeline
+
+When `search(query)` is called:
+
+```
+query
+  ├─ FTS5 search (35% of results)
+  │   └─ BM25 ranking with path boost
+  │
+  ├─ Embedding search (65% of results)
+  │   └─ Cosine similarity ranking
+  │
+  └─ Deduplicate by (path + startLine)
+      └─ Return top-k results
+```
+
+If no embedding provider is configured, falls back to FTS-only.
+
+## Database Schema
+
+```sql
+-- File chunks with line ranges
+chunks (
+  id INTEGER PRIMARY KEY,
+  path TEXT,           -- relative file path
+  cacheKey TEXT,       -- SHA-256 hash of file content
+  content TEXT,        -- chunk text
+  startLine INTEGER,
+  endLine INTEGER,
+  idx INTEGER          -- chunk index within file
+)
+
+-- FTS5 virtual table (trigram tokenizer, BM25 ranking)
+fts (path, content)
+
+-- Links FTS entries to chunks
+fts_metadata (
+  id INTEGER PRIMARY KEY,  -- matches fts rowid
+  path TEXT,
+  cacheKey TEXT,
+  chunkId INTEGER → chunks(id)
+)
+
+-- Vector embeddings
+embeddings (
+  id INTEGER PRIMARY KEY,
+  chunkId INTEGER → chunks(id),
+  path TEXT,
+  cacheKey TEXT,
+  vector TEXT,    -- JSON array of floats
+  model TEXT      -- embedding model identifier
+)
+
+-- Tracks indexed files for incremental updates
+index_catalog (
+  path TEXT,
+  cacheKey TEXT,     -- SHA-256 of file content
+  lastUpdated INTEGER,
+  directory TEXT
+)
+```
+
+Storage location: `~/.cdoing/index.sqlite`
+
+## Usage
+
+### As a Tool (`codebase_search`)
+
+The LLM calls this tool to search the codebase:
+
+```
+codebase_search({ query: "authentication middleware" })
+codebase_search({ query: "sendEmail function", directory: "src/services" })
+```
+
+The index is built lazily on first use and refreshed if stale (>1 hour).
+
+### As a Context Provider (`@codebase`)
+
+Users type `@codebase auth middleware` to attach relevant code to their message.
+
+### Programmatic API
+
+```typescript
+import { CodebaseIndexer } from "@cdoing/core";
+
+const indexer = new CodebaseIndexer("/path/to/project");
+
+// Index (incremental)
+await indexer.index((progress) => {
+  console.log(`${progress.phase}: ${progress.message}`);
+});
+
+// Search
+const results = await indexer.search("authentication", 20);
+for (const r of results) {
+  console.log(`${r.path}:${r.startLine} (${r.source}, score: ${r.score})`);
+  console.log(r.content);
+}
+
+// With embeddings
+import { CodebaseIndexer, type EmbeddingProvider } from "@cdoing/core";
+
+const embedder: EmbeddingProvider = {
+  modelId: "text-embedding-3-small",
+  embed: async (texts) => {
+    // Call OpenAI/Ollama/etc.
+    return vectors;
+  },
+};
+
+const indexer = new CodebaseIndexer("/path/to/project", embedder);
+await indexer.index();
+const results = await indexer.search("how does auth work?"); // Uses both FTS + embeddings
+```
+
+## Comparison with Continue.dev
+
+| Feature | Cdoing | Continue |
+|---|---|---|
+| Storage | Single SQLite file | SQLite + LanceDB (separate) |
+| FTS | FTS5 with trigram tokenizer | FTS5 with trigram tokenizer |
+| BM25 path boost | 10x | 10x |
+| Chunking | Regex boundary detection | Tree-sitter AST |
+| Code structure | Heuristic (function/class patterns) | Full AST parsing (15+ languages) |
+| Embeddings | SQLite JSON + in-process cosine sim | LanceDB native vector search |
+| Incremental updates | SHA-256 content hashing | SHA-256 with cross-branch dedup |
+| Cross-branch cache | Not yet | Content-addressed global cache |
+| Search pipeline | FTS(35%) + Embeddings(65%) | FTS(25%) + Embeddings(50%) + Recent(25%) |
+| Dependencies | better-sqlite3 only | better-sqlite3 + LanceDB native |
+
+### Design Trade-offs
+
+**Why SQLite for embeddings instead of LanceDB?**
+- Zero additional native dependencies (LanceDB requires platform-specific binaries)
+- Single file for all index data
+- In-process cosine similarity is fast enough for codebases < 100k chunks
+- Simpler deployment and no platform compatibility issues
+
+**Why regex chunking instead of tree-sitter?**
+- Tree-sitter requires per-language WASM binaries (~2MB each)
+- Regex patterns cover 90% of function/class boundary detection
+- Falls back gracefully to line-based chunking
+- Much smaller package size
+
+**Future improvements:**
+- Add tree-sitter for more accurate code structure analysis
+- Add recently-edited file cache as a retrieval source
+- Add cross-branch content deduplication
+- Add reranking model support for the retrieval pipeline

package/src/indexing/chunker.ts

@@ -0,0 +1,206 @@
+/**
+ * Code Chunker — splits files into meaningful chunks for indexing.
+ *
+ * Two strategies:
+ *   1. Code-aware: splits on function/class boundaries (heuristic, no tree-sitter)
+ *   2. Basic: splits on line count with overlap
+ *
+ * Inspired by Continue's chunking but without tree-sitter dependency
+ * to keep the package lightweight.
+ */
+
+import * as path from "path";
+
+export interface Chunk {
+  content: string;
+  startLine: number;
+  endLine: number;
+}
+
+/** Max tokens per chunk (~3.5 chars/token, target 384 tokens) */
+const MAX_CHUNK_CHARS = 1400;
+
+/** Overlap between chunks (lines) */
+const OVERLAP_LINES = 3;
+
+/** File extensions that support code-aware chunking */
+const CODE_EXTENSIONS = new Set([
+  ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+  ".py", ".rb", ".go", ".rs", ".java", ".kt",
+  ".c", ".cpp", ".h", ".hpp", ".cs",
+  ".swift", ".scala", ".lua", ".php", ".pl",
+  ".sh", ".bash", ".zsh",
+]);
+
+/** Patterns that indicate function/class boundaries */
+const BOUNDARY_PATTERNS = [
+  /^(?:export\s+)?(?:async\s+)?function\s/,          // function declarations
+  /^(?:export\s+)?(?:const|let|var)\s+\w+\s*=\s*(?:async\s+)?\(/,  // arrow functions
+  /^(?:export\s+)?(?:const|let|var)\s+\w+\s*=\s*(?:async\s+)?function/,
+  /^(?:export\s+)?class\s/,                           // classes
+  /^(?:export\s+)?interface\s/,                        // interfaces
+  /^(?:export\s+)?type\s+\w+\s*=/,                     // type aliases
+  /^(?:export\s+)?enum\s/,                             // enums
+  /^\s*(?:public|private|protected|static|async)\s+\w+\s*\(/, // methods
+  /^def\s+\w+/,                                        // Python functions
+  /^class\s+\w+/,                                      // Python classes
+  /^func\s+/,                                          // Go functions
+  /^(?:pub\s+)?(?:fn|struct|enum|impl|trait)\s/,       // Rust
+  /^(?:public|private|protected)\s+(?:static\s+)?(?:\w+\s+)?\w+\s*\(/, // Java/C#
+];
+
+/**
+ * Check if a file should be chunked.
+ */
+export function shouldChunk(filePath: string, contentLength: number): boolean {
+  if (contentLength === 0) return false;
+  if (contentLength > 1024 * 1024) return false; // Skip files > 1MB
+  const ext = path.extname(filePath).toLowerCase();
+  if (!ext) return false;
+  return true;
+}
+
+/**
+ * Chunk a file's content into meaningful pieces.
+ * Uses code-aware splitting for code files, basic splitting for others.
+ */
+export function chunkDocument(filePath: string, content: string): Chunk[] {
+  if (!content.trim()) return [];
+
+  const ext = path.extname(filePath).toLowerCase();
+
+  if (CODE_EXTENSIONS.has(ext)) {
+    return codeChunker(content);
+  }
+
+  // Markdown: split by headers
+  if (ext === ".md" || ext === ".mdx") {
+    return markdownChunker(content);
+  }
+
+  return basicChunker(content);
+}
+
+/**
+ * Code-aware chunking: split on function/class boundaries.
+ */
+function codeChunker(content: string): Chunk[] {
+  const lines = content.split("\n");
+  const chunks: Chunk[] = [];
+
+  // Find boundary lines
+  const boundaries: number[] = [0];
+  for (let i = 1; i < lines.length; i++) {
+    const trimmed = lines[i].trimStart();
+    if (BOUNDARY_PATTERNS.some((p) => p.test(trimmed))) {
+      boundaries.push(i);
+    }
+  }
+  boundaries.push(lines.length);
+
+  // Create chunks from boundaries
+  for (let i = 0; i < boundaries.length - 1; i++) {
+    const start = boundaries[i];
+    const end = boundaries[i + 1];
+    const chunkLines = lines.slice(start, end);
+    const chunkContent = chunkLines.join("\n").trim();
+
+    if (!chunkContent) continue;
+
+    // If chunk is too large, split it further
+    if (chunkContent.length > MAX_CHUNK_CHARS * 2) {
+      const subChunks = basicChunkerFromLines(chunkLines, start);
+      chunks.push(...subChunks);
+    } else {
+      chunks.push({ content: chunkContent, startLine: start + 1, endLine: end });
+    }
+  }
+
+  // Merge tiny chunks (< 100 chars) with neighbors
+  return mergeSmallChunks(chunks);
+}
+
+/**
+ * Markdown chunking: split by headers.
+ */
+function markdownChunker(content: string): Chunk[] {
+  const lines = content.split("\n");
+  const chunks: Chunk[] = [];
+  let currentStart = 0;
+  let currentLines: string[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    if (/^#{1,3}\s/.test(lines[i]) && currentLines.length > 0) {
+      const text = currentLines.join("\n").trim();
+      if (text) {
+        chunks.push({ content: text, startLine: currentStart + 1, endLine: i });
+      }
+      currentStart = i;
+      currentLines = [lines[i]];
+    } else {
+      currentLines.push(lines[i]);
+    }
+  }
+
+  // Last section
+  const text = currentLines.join("\n").trim();
+  if (text) {
+    chunks.push({ content: text, startLine: currentStart + 1, endLine: lines.length });
+  }
+
+  return mergeSmallChunks(chunks);
+}
+
+/**
+ * Basic line-based chunking with overlap.
+ */
+function basicChunker(content: string): Chunk[] {
+  return basicChunkerFromLines(content.split("\n"), 0);
+}
+
+function basicChunkerFromLines(lines: string[], lineOffset: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  const targetLines = Math.ceil(MAX_CHUNK_CHARS / 80); // ~80 chars per line
+
+  for (let i = 0; i < lines.length; i += targetLines - OVERLAP_LINES) {
+    const end = Math.min(i + targetLines, lines.length);
+    const chunkLines = lines.slice(i, end);
+    const content = chunkLines.join("\n").trim();
+
+    if (content) {
+      chunks.push({
+        content,
+        startLine: lineOffset + i + 1,
+        endLine: lineOffset + end,
+      });
+    }
+
+    if (end >= lines.length) break;
+  }
+
+  return chunks;
+}
+
+function mergeSmallChunks(chunks: Chunk[], minSize = 100): Chunk[] {
+  if (chunks.length <= 1) return chunks;
+
+  const merged: Chunk[] = [];
+  let current = chunks[0];
+
+  for (let i = 1; i < chunks.length; i++) {
+    if (current.content.length < minSize) {
+      // Merge with next
+      current = {
+        content: current.content + "\n\n" + chunks[i].content,
+        startLine: current.startLine,
+        endLine: chunks[i].endLine,
+      };
+    } else {
+      merged.push(current);
+      current = chunks[i];
+    }
+  }
+  merged.push(current);
+
+  return merged;
+}