@outfitter/index 0.1.0-rc.1

package/README.md

@@ -0,0 +1,292 @@
+# @outfitter/index
+
+SQLite FTS5 full-text search indexing with WAL mode and Result-based error handling.
+
+## Installation
+
+```bash
+bun add @outfitter/index
+```
+
+## Quick Start
+
+```typescript
+import { createIndex } from "@outfitter/index";
+
+// Create an index
+const index = createIndex({ path: "./data/search.db" });
+
+// Add documents
+await index.add({
+  id: "doc-1",
+  content: "Hello world, this is searchable content",
+  metadata: { title: "Greeting", tags: ["hello", "world"] },
+});
+
+// Search with FTS5 syntax
+const results = await index.search({ query: "hello" });
+
+if (results.isOk()) {
+  for (const result of results.value) {
+    console.log(result.id, result.score, result.highlights);
+  }
+}
+
+// Cleanup
+index.close();
+```
+
+## Features
+
+- **FTS5 Full-Text Search** — BM25 ranking with snippet highlights
+- **WAL Mode** — Better concurrency for read-heavy workloads
+- **Typed Metadata** — Generic type parameter for document metadata
+- **Result-Based API** — All operations return `Result<T, StorageError>`
+- **Tokenizer Options** — unicode61, porter (stemming), or trigram
+- **Batch Operations** — Efficient bulk document insertion
+- **Version Migration** — Built-in schema migration support
+
+## API Reference
+
+### createIndex(options)
+
+Creates an FTS5 full-text search index.
+
+```typescript
+interface IndexOptions {
+  path: string;                   // Path to SQLite database file
+  tableName?: string;             // FTS5 table name (default: "documents")
+  tokenizer?: TokenizerType;      // Tokenizer (default: "unicode61")
+  tool?: string;                  // Tool identifier for metadata
+  toolVersion?: string;           // Tool version for metadata
+  migrations?: IndexMigrationRegistry;  // Optional migration registry
+}
+
+const index = createIndex<MyMetadata>({
+  path: "./data/index.db",
+  tableName: "notes_fts",
+  tokenizer: "porter",
+});
+```
+
+### Tokenizer Types
+
+| Tokenizer | Use Case |
+|-----------|----------|
+| `unicode61` | Default, Unicode-aware word tokenization |
+| `porter` | English text with stemming (finds "running" when searching "run") |
+| `trigram` | Substring matching, typo tolerance |
+
+### Index Methods
+
+```typescript
+interface Index<T = unknown> {
+  // Add single document (replaces if ID exists)
+  add(doc: IndexDocument): Promise<Result<void, StorageError>>;
+
+  // Add multiple documents in a transaction
+  addMany(docs: IndexDocument[]): Promise<Result<void, StorageError>>;
+
+  // Search with FTS5 query syntax
+  search(query: SearchQuery): Promise<Result<SearchResult<T>[], StorageError>>;
+
+  // Remove document by ID
+  remove(id: string): Promise<Result<void, StorageError>>;
+
+  // Clear all documents
+  clear(): Promise<Result<void, StorageError>>;
+
+  // Close database connection
+  close(): void;
+}
+```
+
+### Document Structure
+
+```typescript
+interface IndexDocument {
+  id: string;                           // Unique document ID
+  content: string;                      // Searchable text
+  metadata?: Record<string, unknown>;   // Optional metadata (stored as JSON)
+}
+
+await index.add({
+  id: "note-123",
+  content: "Meeting notes from standup",
+  metadata: {
+    title: "Standup Notes",
+    date: "2024-01-15",
+    tags: ["meeting", "standup"],
+  },
+});
+```
+
+### Search Query
+
+```typescript
+interface SearchQuery {
+  query: string;    // FTS5 query string
+  limit?: number;   // Max results (default: 25)
+  offset?: number;  // Skip results for pagination (default: 0)
+}
+
+// Simple search
+const results = await index.search({ query: "typescript" });
+
+// Phrase search with pagination
+const paged = await index.search({
+  query: '"error handling"',
+  limit: 10,
+  offset: 20,
+});
+```
+
+### FTS5 Query Syntax
+
+FTS5 supports powerful query syntax:
+
+| Syntax | Example | Description |
+|--------|---------|-------------|
+| Terms | `typescript bun` | Match all terms (implicit AND) |
+| Phrase | `"error handling"` | Exact phrase match |
+| OR | `ts OR typescript` | Match either term |
+| NOT | `typescript NOT javascript` | Exclude term |
+| Prefix | `type*` | Prefix matching |
+| Grouping | `(react OR vue) AND typescript` | Complex queries |
+
+### Search Results
+
+```typescript
+interface SearchResult<T = unknown> {
+  id: string;           // Document ID
+  content: string;      // Full document content
+  score: number;        // BM25 relevance (negative; closer to 0 = better match)
+  metadata?: T;         // Document metadata
+  highlights?: string[];  // Matching snippets with <b> tags
+}
+
+const results = await index.search({ query: "hello world" });
+
+if (results.isOk()) {
+  for (const result of results.value) {
+    console.log(`${result.id}: ${result.highlights?.[0]}`);
+    // "doc-1: <b>Hello</b> <b>world</b>, this is..."
+  }
+}
+```
+
+## Batch Operations
+
+For bulk indexing, use `addMany` for transactional efficiency:
+
+```typescript
+const documents = [
+  { id: "1", content: "First document" },
+  { id: "2", content: "Second document" },
+  { id: "3", content: "Third document" },
+];
+
+const result = await index.addMany(documents);
+
+if (result.isErr()) {
+  // Transaction rolled back, no documents added
+  console.error(result.error.message);
+}
+```
+
+## Version Migration
+
+Indexes track their schema version. Provide a migration registry for upgrades:
+
+```typescript
+import { createIndex, createMigrationRegistry } from "@outfitter/index";
+
+const migrations = createMigrationRegistry();
+
+migrations.register(1, 2, (ctx) => {
+  ctx.db.run("ALTER TABLE documents ADD COLUMN category TEXT");
+  return Result.ok(undefined);
+});
+
+const index = createIndex({
+  path: "./data/index.db",
+  migrations,
+});
+```
+
+### Migration Registry
+
+```typescript
+interface IndexMigrationRegistry {
+  register(
+    fromVersion: number,
+    toVersion: number,
+    migrate: (ctx: IndexMigrationContext) => Result<void, StorageError>
+  ): void;
+
+  migrate(
+    ctx: IndexMigrationContext,
+    fromVersion: number,
+    toVersion: number
+  ): Result<void, StorageError>;
+}
+
+interface IndexMigrationContext {
+  db: Database;  // bun:sqlite Database instance
+}
+```
+
+## Index Metadata
+
+Indexes store metadata for tracking provenance:
+
+```typescript
+interface IndexMetadata {
+  version: number;      // Schema version
+  created: string;      // ISO timestamp
+  tool: string;         // Creating tool identifier
+  toolVersion: string;  // Creating tool version
+}
+```
+
+## Version Constant
+
+The current index format version is exported for compatibility checks:
+
+```typescript
+import { INDEX_VERSION } from "@outfitter/index";
+
+console.log(`Using index format version ${INDEX_VERSION}`);
+```
+
+## Error Handling
+
+All operations return `Result<T, StorageError>`:
+
+```typescript
+const result = await index.add(doc);
+
+if (result.isErr()) {
+  console.error("Failed to add document:", result.error.message);
+  // result.error.cause contains the underlying error
+}
+```
+
+Common error scenarios:
+- Index closed after `close()` called
+- Invalid table name or tokenizer
+- SQLite errors (disk full, permissions)
+- Version mismatch without migrations
+
+## Performance Tips
+
+1. **Use WAL mode** — Enabled by default for better read concurrency
+2. **Batch inserts** — Use `addMany` for bulk operations
+3. **Choose tokenizer wisely** — `porter` for English, `unicode61` for general use
+4. **Limit results** — Use pagination for large result sets
+5. **Close when done** — Call `close()` to release resources
+
+## Related Packages
+
+- [@outfitter/contracts](../contracts/README.md) — Result types and StorageError
+- [@outfitter/file-ops](../file-ops/README.md) — Path utilities and workspace detection

package/dist/fts5.d.ts

@@ -0,0 +1,286 @@
+import { Database } from "bun:sqlite";
+import { StorageError } from "@outfitter/contracts";
+import { Result } from "@outfitter/contracts";
+interface MigrationRegistry<TContext> {
+	register(fromVersion: number, toVersion: number, migrate: (context: TContext) => Result<void, StorageError>): void;
+	migrate(context: TContext, fromVersion: number, toVersion: number): Result<void, StorageError>;
+}
+interface IndexMigrationContext {
+	readonly db: Database;
+}
+type IndexMigrationRegistry = MigrationRegistry<IndexMigrationContext>;
+import { Result as Result2, StorageError as StorageError2 } from "@outfitter/contracts";
+/**
+* FTS5 tokenizer options for text analysis.
+*
+* - `unicode61`: Default tokenizer with Unicode support (recommended for most use cases)
+* - `porter`: Applies Porter stemming algorithm for English text (finds related word forms)
+* - `trigram`: Splits text into 3-character sequences (good for substring matching)
+*/
+type TokenizerType = "unicode61" | "porter" | "trigram";
+/**
+* Options for creating an FTS5 index.
+*
+* @example
+* ```typescript
+* const options: IndexOptions = {
+*   path: "/path/to/index.db",
+*   tableName: "documents",
+*   tokenizer: "porter",
+* };
+*
+* const index = createIndex(options);
+* ```
+*/
+interface IndexOptions {
+	/**
+	* Absolute path to the SQLite database file.
+	* The file will be created if it does not exist.
+	*/
+	path: string;
+	/**
+	* Name of the FTS5 virtual table.
+	* @defaultValue "documents"
+	*/
+	tableName?: string;
+	/**
+	* FTS5 tokenizer for text analysis.
+	* @defaultValue "unicode61"
+	*/
+	tokenizer?: TokenizerType;
+	/**
+	* Optional tool identifier recorded in index metadata.
+	*/
+	tool?: string;
+	/**
+	* Optional tool version recorded in index metadata.
+	*/
+	toolVersion?: string;
+	/**
+	* Optional migration registry for upgrading older index versions.
+	*/
+	migrations?: IndexMigrationRegistry;
+}
+/**
+* A document to be indexed in the FTS5 index.
+*
+* Documents have a unique ID, searchable content, and optional metadata.
+* The metadata is stored as JSON and can be used to attach additional
+* information that is returned with search results.
+*
+* @example
+* ```typescript
+* const doc: IndexDocument = {
+*   id: "note-123",
+*   content: "This is the searchable text content",
+*   metadata: { title: "My Note", createdAt: Date.now() },
+* };
+* ```
+*/
+interface IndexDocument {
+	/** Unique identifier for this document */
+	id: string;
+	/** Searchable text content */
+	content: string;
+	/**
+	* Optional metadata associated with the document.
+	* Stored as JSON and returned with search results.
+	*/
+	metadata?: Record<string, unknown>;
+}
+/**
+* Query parameters for searching the FTS5 index.
+*
+* Uses FTS5 query syntax which supports:
+* - Simple terms: `search term`
+* - Phrases: `"exact phrase"`
+* - Boolean operators: `term1 AND term2`, `term1 OR term2`, `NOT term`
+* - Prefix matching: `term*`
+* - Grouping: `(term1 OR term2) AND term3`
+*
+* @example
+* ```typescript
+* // Simple search
+* const query1: SearchQuery = { query: "typescript" };
+*
+* // Phrase search with pagination
+* const query2: SearchQuery = {
+*   query: '"error handling"',
+*   limit: 10,
+*   offset: 20,
+* };
+* ```
+*/
+interface SearchQuery {
+	/** FTS5 query string */
+	query: string;
+	/**
+	* Maximum number of results to return.
+	* @defaultValue 25
+	*/
+	limit?: number;
+	/**
+	* Number of results to skip (for pagination).
+	* @defaultValue 0
+	*/
+	offset?: number;
+}
+/**
+* A single search result from an FTS5 query.
+*
+* Results include the document ID, BM25 relevance score, content,
+* and any associated metadata. Optional highlights show matching
+* snippets from the content.
+*
+* @typeParam T - Type of the metadata (defaults to `unknown`)
+*
+* @example
+* ```typescript
+* interface NoteMetadata {
+*   title: string;
+*   tags: string[];
+* }
+*
+* const result: SearchResult<NoteMetadata> = {
+*   id: "note-123",
+*   score: 0.85,
+*   content: "Full document content...",
+*   metadata: { title: "My Note", tags: ["typescript"] },
+*   highlights: ["...matching <b>snippet</b>..."],
+* };
+* ```
+*/
+interface SearchResult<T = unknown> {
+	/** Document ID */
+	id: string;
+	/**
+	* BM25 relevance ranking score.
+	* Higher scores indicate better matches.
+	* Note: FTS5 BM25 returns negative values (closer to 0 = better match).
+	*/
+	score: number;
+	/** Full document content */
+	content: string;
+	/** Document metadata (if present) */
+	metadata?: T;
+	/**
+	* Matching snippets from the content.
+	* Uses FTS5 snippet() function for context-aware highlights.
+	*/
+	highlights?: string[];
+}
+/**
+* The FTS5 index interface for full-text search operations.
+*
+* Provides methods for adding, searching, and removing documents
+* from an SQLite FTS5 index. All operations return `Result` types
+* for explicit error handling.
+*
+* @typeParam T - Type of document metadata (defaults to `unknown`)
+*
+* @example
+* ```typescript
+* const index = createIndex<NoteMetadata>({ path: "./index.db" });
+*
+* // Add documents
+* await index.add({ id: "1", content: "Hello world", metadata: { title: "Greeting" } });
+*
+* // Search
+* const results = await index.search({ query: "hello" });
+* if (results.isOk()) {
+*   for (const result of results.value) {
+*     console.log(result.id, result.score);
+*   }
+* }
+*
+* // Cleanup
+* index.close();
+* ```
+*/
+interface Index<T = unknown> {
+	/**
+	* Add a single document to the index.
+	* If a document with the same ID exists, it will be replaced.
+	*
+	* @param doc - Document to add
+	* @returns Result indicating success or StorageError
+	*/
+	add(doc: IndexDocument): Promise<Result2<void, StorageError2>>;
+	/**
+	* Add multiple documents to the index in a single transaction.
+	* More efficient than calling add() multiple times.
+	* If a document with the same ID exists, it will be replaced.
+	*
+	* @param docs - Array of documents to add
+	* @returns Result indicating success or StorageError
+	*/
+	addMany(docs: IndexDocument[]): Promise<Result2<void, StorageError2>>;
+	/**
+	* Search the index using FTS5 query syntax.
+	* Returns results ranked by BM25 relevance score.
+	*
+	* @param query - Search query parameters
+	* @returns Result containing array of search results or StorageError
+	*/
+	search(query: SearchQuery): Promise<Result2<SearchResult<T>[], StorageError2>>;
+	/**
+	* Remove a document from the index by ID.
+	* No error is returned if the document does not exist.
+	*
+	* @param id - Document ID to remove
+	* @returns Result indicating success or StorageError
+	*/
+	remove(id: string): Promise<Result2<void, StorageError2>>;
+	/**
+	* Remove all documents from the index.
+	*
+	* @returns Result indicating success or StorageError
+	*/
+	clear(): Promise<Result2<void, StorageError2>>;
+	/**
+	* Close the index and release resources.
+	* The index should not be used after calling close().
+	*/
+	close(): void;
+}
+/**
+* Creates an FTS5 full-text search index.
+*
+* Uses SQLite FTS5 virtual table for fast full-text search with BM25 ranking.
+* The database is configured with WAL mode for better concurrency.
+*
+* @typeParam T - Type of document metadata (defaults to `unknown`)
+* @param options - Index options including database path and table configuration
+* @returns An Index instance for managing documents and searching
+*
+* @example
+* ```typescript
+* // Create an index with default settings
+* const index = createIndex({ path: "./data/index.db" });
+*
+* // Add documents
+* await index.add({
+*   id: "doc-1",
+*   content: "Hello world",
+*   metadata: { title: "Greeting" },
+* });
+*
+* // Search
+* const results = await index.search({ query: "hello" });
+*
+* // Cleanup
+* index.close();
+* ```
+*
+* @example
+* ```typescript
+* // Create an index with Porter stemmer for English text
+* const index = createIndex({
+*   path: "./data/notes.db",
+*   tableName: "notes_fts",
+*   tokenizer: "porter",
+* });
+* ```
+*/
+declare function createIndex<T = unknown>(options: IndexOptions): Index<T>;
+export { createIndex };