@outfitter/index 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,12 +54,12 @@ Creates an FTS5 full-text search index.
54
54
 
55
55
  ```typescript
56
56
  interface IndexOptions {
57
- path: string; // Path to SQLite database file
58
- tableName?: string; // FTS5 table name (default: "documents")
59
- tokenizer?: TokenizerType; // Tokenizer (default: "unicode61")
60
- tool?: string; // Tool identifier for metadata
61
- toolVersion?: string; // Tool version for metadata
62
- migrations?: IndexMigrationRegistry; // Optional migration registry
57
+ path: string; // Path to SQLite database file
58
+ tableName?: string; // FTS5 table name (default: "documents")
59
+ tokenizer?: TokenizerType; // Tokenizer (default: "unicode61")
60
+ tool?: string; // Tool identifier for metadata
61
+ toolVersion?: string; // Tool version for metadata
62
+ migrations?: IndexMigrationRegistry; // Optional migration registry
63
63
  }
64
64
 
65
65
  const index = createIndex<MyMetadata>({
@@ -71,11 +71,11 @@ const index = createIndex<MyMetadata>({
71
71
 
72
72
  ### Tokenizer Types
73
73
 
74
- | Tokenizer | Use Case |
75
- |-----------|----------|
76
- | `unicode61` | Default, Unicode-aware word tokenization |
77
- | `porter` | English text with stemming (finds "running" when searching "run") |
78
- | `trigram` | Substring matching, typo tolerance |
74
+ | Tokenizer | Use Case |
75
+ | ----------- | ----------------------------------------------------------------- |
76
+ | `unicode61` | Default, Unicode-aware word tokenization |
77
+ | `porter` | English text with stemming (finds "running" when searching "run") |
78
+ | `trigram` | Substring matching, typo tolerance |
79
79
 
80
80
  ### Index Methods
81
81
 
@@ -105,9 +105,9 @@ interface Index<T = unknown> {
105
105
 
106
106
  ```typescript
107
107
  interface IndexDocument {
108
- id: string; // Unique document ID
109
- content: string; // Searchable text
110
- metadata?: Record<string, unknown>; // Optional metadata (stored as JSON)
108
+ id: string; // Unique document ID
109
+ content: string; // Searchable text
110
+ metadata?: Record<string, unknown>; // Optional metadata (stored as JSON)
111
111
  }
112
112
 
113
113
  await index.add({
@@ -125,9 +125,9 @@ await index.add({
125
125
 
126
126
  ```typescript
127
127
  interface SearchQuery {
128
- query: string; // FTS5 query string
129
- limit?: number; // Max results (default: 25)
130
- offset?: number; // Skip results for pagination (default: 0)
128
+ query: string; // FTS5 query string
129
+ limit?: number; // Max results (default: 25)
130
+ offset?: number; // Skip results for pagination (default: 0)
131
131
  }
132
132
 
133
133
  // Simple search
@@ -145,24 +145,24 @@ const paged = await index.search({
145
145
 
146
146
  FTS5 supports powerful query syntax:
147
147
 
148
- | Syntax | Example | Description |
149
- |--------|---------|-------------|
150
- | Terms | `typescript bun` | Match all terms (implicit AND) |
151
- | Phrase | `"error handling"` | Exact phrase match |
152
- | OR | `ts OR typescript` | Match either term |
153
- | NOT | `typescript NOT javascript` | Exclude term |
154
- | Prefix | `type*` | Prefix matching |
155
- | Grouping | `(react OR vue) AND typescript` | Complex queries |
148
+ | Syntax | Example | Description |
149
+ | -------- | ------------------------------- | ------------------------------ |
150
+ | Terms | `typescript bun` | Match all terms (implicit AND) |
151
+ | Phrase | `"error handling"` | Exact phrase match |
152
+ | OR | `ts OR typescript` | Match either term |
153
+ | NOT | `typescript NOT javascript` | Exclude term |
154
+ | Prefix | `type*` | Prefix matching |
155
+ | Grouping | `(react OR vue) AND typescript` | Complex queries |
156
156
 
157
157
  ### Search Results
158
158
 
159
159
  ```typescript
160
160
  interface SearchResult<T = unknown> {
161
- id: string; // Document ID
162
- content: string; // Full document content
163
- score: number; // BM25 relevance (negative; closer to 0 = better match)
164
- metadata?: T; // Document metadata
165
- highlights?: string[]; // Matching snippets with <b> tags
161
+ id: string; // Document ID
162
+ content: string; // Full document content
163
+ score: number; // BM25 relevance (negative; closer to 0 = better match)
164
+ metadata?: T; // Document metadata
165
+ highlights?: string[]; // Matching snippets with <b> tags
166
166
  }
167
167
 
168
168
  const results = await index.search({ query: "hello world" });
@@ -232,7 +232,7 @@ interface IndexMigrationRegistry {
232
232
  }
233
233
 
234
234
  interface IndexMigrationContext {
235
- db: Database; // bun:sqlite Database instance
235
+ db: Database; // bun:sqlite Database instance
236
236
  }
237
237
  ```
238
238
 
@@ -242,10 +242,10 @@ Indexes store metadata for tracking provenance:
242
242
 
243
243
  ```typescript
244
244
  interface IndexMetadata {
245
- version: number; // Schema version
246
- created: string; // ISO timestamp
247
- tool: string; // Creating tool identifier
248
- toolVersion: string; // Creating tool version
245
+ version: number; // Schema version
246
+ created: string; // ISO timestamp
247
+ tool: string; // Creating tool identifier
248
+ toolVersion: string; // Creating tool version
249
249
  }
250
250
  ```
251
251
 
@@ -273,6 +273,7 @@ if (result.isErr()) {
273
273
  ```
274
274
 
275
275
  Common error scenarios:
276
+
276
277
  - Index closed after `close()` called
277
278
  - Invalid table name or tokenizer
278
279
  - SQLite errors (disk full, permissions)
package/dist/fts5.js CHANGED
@@ -1,8 +1,252 @@
1
1
  // @bun
2
2
  import {
3
- createIndex
4
- } from "./shared/@outfitter/index-gf30ny51.js";
5
- import"./shared/@outfitter/index-bbzmc40h.js";
3
+ INDEX_META_KEY,
4
+ INDEX_META_TABLE,
5
+ INDEX_VERSION
6
+ } from "./shared/@outfitter/index-bbzmc40h.js";
7
+
8
+ // packages/index/src/fts5.ts
9
+ import { Database } from "bun:sqlite";
10
+ import { existsSync, mkdirSync } from "fs";
11
+ import { dirname } from "path";
12
+ import { Result } from "@outfitter/contracts";
13
+ var DEFAULT_TABLE_NAME = "documents";
14
+ var DEFAULT_TOKENIZER = "unicode61";
15
+ var DEFAULT_LIMIT = 25;
16
+ var DEFAULT_OFFSET = 0;
17
+ var VALID_TOKENIZERS = {
18
+ unicode61: true,
19
+ porter: true,
20
+ trigram: true
21
+ };
22
+ var TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
23
+ var DEFAULT_TOOL = "outfitter-index";
24
+ var DEFAULT_TOOL_VERSION = "0.0.0";
25
+ function createStorageError(message, cause) {
26
+ return {
27
+ _tag: "StorageError",
28
+ message,
29
+ cause
30
+ };
31
+ }
32
+ function assertValidTableName(tableName) {
33
+ if (!TABLE_NAME_PATTERN.test(tableName)) {
34
+ throw new Error(`Invalid table name: ${tableName}`);
35
+ }
36
+ }
37
+ function assertValidTokenizer(tokenizer) {
38
+ if (!Object.hasOwn(VALID_TOKENIZERS, tokenizer)) {
39
+ throw new Error(`Invalid tokenizer: ${tokenizer}`);
40
+ }
41
+ return tokenizer;
42
+ }
43
+ function getUserVersion(db) {
44
+ const row = db.query("PRAGMA user_version").get();
45
+ return row?.user_version ?? 0;
46
+ }
47
+ function setUserVersion(db, version) {
48
+ if (!Number.isInteger(version) || version < 0) {
49
+ throw new Error(`Invalid user_version: ${version}`);
50
+ }
51
+ db.run(`PRAGMA user_version = ${version}`);
52
+ }
53
+ function ensureMetaTable(db) {
54
+ db.run(`CREATE TABLE IF NOT EXISTS ${INDEX_META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
55
+ }
56
+ function readIndexMetadata(db) {
57
+ try {
58
+ const row = db.query(`SELECT value FROM ${INDEX_META_TABLE} WHERE key = ?`).get(INDEX_META_KEY);
59
+ if (!row) {
60
+ return null;
61
+ }
62
+ const parsed = JSON.parse(row.value);
63
+ return parsed;
64
+ } catch {
65
+ return null;
66
+ }
67
+ }
68
+ function writeIndexMetadata(db, metadata) {
69
+ ensureMetaTable(db);
70
+ db.run(`INSERT OR REPLACE INTO ${INDEX_META_TABLE} (key, value) VALUES (?, ?)`, [INDEX_META_KEY, JSON.stringify(metadata)]);
71
+ }
72
+ function createIndex(options) {
73
+ const tableName = options.tableName ?? DEFAULT_TABLE_NAME;
74
+ assertValidTableName(tableName);
75
+ const tokenizer = assertValidTokenizer(options.tokenizer ?? DEFAULT_TOKENIZER);
76
+ const tool = options.tool ?? DEFAULT_TOOL;
77
+ const toolVersion = options.toolVersion ?? DEFAULT_TOOL_VERSION;
78
+ const dir = dirname(options.path);
79
+ if (!existsSync(dir)) {
80
+ mkdirSync(dir, { recursive: true });
81
+ }
82
+ const db = new Database(options.path);
83
+ db.run("PRAGMA journal_mode=WAL");
84
+ const currentVersion = getUserVersion(db);
85
+ if (currentVersion === 0) {
86
+ const metadata = {
87
+ version: INDEX_VERSION,
88
+ created: new Date().toISOString(),
89
+ tool,
90
+ toolVersion
91
+ };
92
+ setUserVersion(db, INDEX_VERSION);
93
+ writeIndexMetadata(db, metadata);
94
+ } else if (currentVersion !== INDEX_VERSION) {
95
+ if (!options.migrations) {
96
+ throw new Error(`Index version ${currentVersion} does not match ${INDEX_VERSION}. Provide migrations or rebuild the index.`);
97
+ }
98
+ const context = { db };
99
+ const result = options.migrations.migrate(context, currentVersion, INDEX_VERSION);
100
+ if (result.isErr()) {
101
+ throw new Error(`Failed to migrate index: ${result.error.message}`);
102
+ }
103
+ const existing = readIndexMetadata(db);
104
+ const metadata = {
105
+ version: INDEX_VERSION,
106
+ created: existing?.created ?? new Date().toISOString(),
107
+ tool,
108
+ toolVersion
109
+ };
110
+ setUserVersion(db, INDEX_VERSION);
111
+ writeIndexMetadata(db, metadata);
112
+ } else if (!readIndexMetadata(db)) {
113
+ const metadata = {
114
+ version: INDEX_VERSION,
115
+ created: new Date().toISOString(),
116
+ tool,
117
+ toolVersion
118
+ };
119
+ writeIndexMetadata(db, metadata);
120
+ }
121
+ db.run(`
122
+ CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName}
123
+ USING fts5(
124
+ id UNINDEXED,
125
+ content,
126
+ metadata UNINDEXED,
127
+ tokenize='${tokenizer}'
128
+ )
129
+ `);
130
+ let isClosed = false;
131
+ function checkClosed() {
132
+ if (isClosed) {
133
+ return Result.err(createStorageError("Index is closed"));
134
+ }
135
+ return Result.ok(undefined);
136
+ }
137
+ return {
138
+ async add(doc) {
139
+ const closedCheck = checkClosed();
140
+ if (closedCheck.isErr()) {
141
+ return closedCheck;
142
+ }
143
+ try {
144
+ const metadataJson = doc.metadata ? JSON.stringify(doc.metadata) : null;
145
+ db.run(`DELETE FROM ${tableName} WHERE id = ?`, [doc.id]);
146
+ db.run(`INSERT INTO ${tableName} (id, content, metadata) VALUES (?, ?, ?)`, [doc.id, doc.content, metadataJson]);
147
+ return Result.ok(undefined);
148
+ } catch (error) {
149
+ return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to add document", error));
150
+ }
151
+ },
152
+ async addMany(docs) {
153
+ const closedCheck = checkClosed();
154
+ if (closedCheck.isErr()) {
155
+ return closedCheck;
156
+ }
157
+ try {
158
+ db.run("BEGIN TRANSACTION");
159
+ try {
160
+ const deleteStmt = db.prepare(`DELETE FROM ${tableName} WHERE id = ?`);
161
+ const insertStmt = db.prepare(`INSERT INTO ${tableName} (id, content, metadata) VALUES (?, ?, ?)`);
162
+ for (const doc of docs) {
163
+ const metadataJson = doc.metadata ? JSON.stringify(doc.metadata) : null;
164
+ deleteStmt.run(doc.id);
165
+ insertStmt.run(doc.id, doc.content, metadataJson);
166
+ }
167
+ deleteStmt.finalize();
168
+ insertStmt.finalize();
169
+ db.run("COMMIT");
170
+ return Result.ok(undefined);
171
+ } catch (error) {
172
+ db.run("ROLLBACK");
173
+ throw error;
174
+ }
175
+ } catch (error) {
176
+ return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to add documents", error));
177
+ }
178
+ },
179
+ async search(query) {
180
+ const closedCheck = checkClosed();
181
+ if (closedCheck.isErr()) {
182
+ return closedCheck;
183
+ }
184
+ const limit = query.limit ?? DEFAULT_LIMIT;
185
+ const offset = query.offset ?? DEFAULT_OFFSET;
186
+ try {
187
+ const rows = db.query(`
188
+ SELECT
189
+ id,
190
+ content,
191
+ metadata,
192
+ bm25(${tableName}) as score,
193
+ snippet(${tableName}, 1, '<b>', '</b>', '...', 32) as highlight
194
+ FROM ${tableName}
195
+ WHERE ${tableName} MATCH ?
196
+ ORDER BY bm25(${tableName}) ASC
197
+ LIMIT ? OFFSET ?
198
+ `).all(query.query, limit, offset);
199
+ const results = rows.map((row) => {
200
+ const result = {
201
+ id: row.id,
202
+ content: row.content,
203
+ score: row.score,
204
+ highlights: [row.highlight]
205
+ };
206
+ if (row.metadata) {
207
+ try {
208
+ result.metadata = JSON.parse(row.metadata);
209
+ } catch {}
210
+ }
211
+ return result;
212
+ });
213
+ return Result.ok(results);
214
+ } catch (error) {
215
+ return Result.err(createStorageError(error instanceof Error ? error.message : "Search failed", error));
216
+ }
217
+ },
218
+ async remove(id) {
219
+ const closedCheck = checkClosed();
220
+ if (closedCheck.isErr()) {
221
+ return closedCheck;
222
+ }
223
+ try {
224
+ db.run(`DELETE FROM ${tableName} WHERE id = ?`, [id]);
225
+ return Result.ok(undefined);
226
+ } catch (error) {
227
+ return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to remove document", error));
228
+ }
229
+ },
230
+ async clear() {
231
+ const closedCheck = checkClosed();
232
+ if (closedCheck.isErr()) {
233
+ return closedCheck;
234
+ }
235
+ try {
236
+ db.run(`DELETE FROM ${tableName}`);
237
+ return Result.ok(undefined);
238
+ } catch (error) {
239
+ return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to clear index", error));
240
+ }
241
+ },
242
+ close() {
243
+ if (!isClosed) {
244
+ isClosed = true;
245
+ db.close();
246
+ }
247
+ }
248
+ };
249
+ }
6
250
  export {
7
251
  createIndex
8
252
  };
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createIndex } from "./shared/@outfitter/index-ykxa3yhb.js";
2
- import { INDEX_VERSION } from "./shared/@outfitter/index-3xe3cd6r.js";
3
2
  import { Index, IndexDocument, IndexMetadata, IndexOptions, SearchQuery, SearchResult, TokenizerType } from "./shared/@outfitter/index-011azjav.js";
3
+ import { INDEX_VERSION } from "./shared/@outfitter/index-3xe3cd6r.js";
4
4
  import { IndexMigrationRegistry, createMigrationRegistry } from "./shared/@outfitter/index-dxk50kvh.js";
5
5
  export { createMigrationRegistry, createIndex, TokenizerType, SearchResult, SearchQuery, IndexOptions, IndexMigrationRegistry, IndexMetadata, IndexDocument, Index, INDEX_VERSION };
package/dist/index.js CHANGED
@@ -1,16 +1,4 @@
1
- // @bun
2
- import {
3
- createIndex
4
- } from "./shared/@outfitter/index-gf30ny51.js";
5
- import {
6
- createMigrationRegistry
7
- } from "./shared/@outfitter/index-1me624ny.js";
8
- import {
9
- INDEX_VERSION
10
- } from "./shared/@outfitter/index-bbzmc40h.js";
11
- import"./shared/@outfitter/index-8fwmfq7d.js";
12
- export {
13
- createMigrationRegistry,
14
- createIndex,
15
- INDEX_VERSION
16
- };
1
+ export { createIndex } from "./fts5.js";
2
+ export { createMigrationRegistry } from "./migrations.js";
3
+ export * from "./types.js";
4
+ export { INDEX_VERSION } from "./version.js";
@@ -1,7 +1,47 @@
1
1
  // @bun
2
- import {
3
- createMigrationRegistry
4
- } from "./shared/@outfitter/index-1me624ny.js";
2
+ // packages/index/src/migrations.ts
3
+ import { Result } from "@outfitter/contracts";
4
+ function createStorageError(message, cause) {
5
+ return {
6
+ _tag: "StorageError",
7
+ message,
8
+ cause
9
+ };
10
+ }
11
+ function createMigrationRegistry() {
12
+ const steps = new Map;
13
+ return {
14
+ register(fromVersion, toVersion, migrate) {
15
+ steps.set(fromVersion, { to: toVersion, migrate });
16
+ },
17
+ migrate(context, fromVersion, toVersion) {
18
+ if (fromVersion === toVersion) {
19
+ return Result.ok(undefined);
20
+ }
21
+ let current = fromVersion;
22
+ const visited = new Set;
23
+ while (current < toVersion) {
24
+ if (visited.has(current)) {
25
+ return Result.err(createStorageError(`Detected migration loop at version ${current}`));
26
+ }
27
+ visited.add(current);
28
+ const step = steps.get(current);
29
+ if (!step) {
30
+ return Result.err(createStorageError(`No migration registered from version ${current}`));
31
+ }
32
+ const result = step.migrate(context);
33
+ if (result.isErr()) {
34
+ return result;
35
+ }
36
+ current = step.to;
37
+ }
38
+ if (current !== toVersion) {
39
+ return Result.err(createStorageError(`Migration ended at version ${current} instead of ${toVersion}`));
40
+ }
41
+ return Result.ok(undefined);
42
+ }
43
+ };
44
+ }
5
45
  export {
6
46
  createMigrationRegistry
7
47
  };
package/package.json CHANGED
@@ -1,11 +1,26 @@
1
1
  {
2
2
  "name": "@outfitter/index",
3
+ "version": "0.2.4",
3
4
  "description": "SQLite FTS5 full-text search indexing for Outfitter",
4
- "version": "0.2.3",
5
- "type": "module",
5
+ "keywords": [
6
+ "fts5",
7
+ "index",
8
+ "outfitter",
9
+ "search",
10
+ "sqlite",
11
+ "typescript"
12
+ ],
13
+ "license": "MIT",
14
+ "repository": {
15
+ "type": "git",
16
+ "url": "https://github.com/outfitter-dev/outfitter.git",
17
+ "directory": "packages/index"
18
+ },
6
19
  "files": [
7
20
  "dist"
8
21
  ],
22
+ "type": "module",
23
+ "sideEffects": false,
9
24
  "module": "./dist/index.js",
10
25
  "types": "./dist/index.d.ts",
11
26
  "exports": {
@@ -41,39 +56,24 @@
41
56
  }
42
57
  }
43
58
  },
44
- "sideEffects": false,
59
+ "publishConfig": {
60
+ "access": "public"
61
+ },
45
62
  "scripts": {
46
63
  "build": "cd ../.. && bunup --filter @outfitter/index",
47
- "lint": "biome lint ./src",
48
- "lint:fix": "biome lint --write ./src",
64
+ "lint": "oxlint ./src",
65
+ "lint:fix": "oxlint --fix ./src",
49
66
  "test": "bun test",
50
67
  "typecheck": "tsc --noEmit",
51
68
  "clean": "rm -rf dist",
52
69
  "prepublishOnly": "bun ../../scripts/check-publish-manifest.ts"
53
70
  },
54
71
  "dependencies": {
55
- "@outfitter/contracts": "0.4.1",
56
- "@outfitter/file-ops": "0.2.3"
72
+ "@outfitter/contracts": "0.4.2",
73
+ "@outfitter/file-ops": "0.2.4"
57
74
  },
58
75
  "devDependencies": {
59
- "@types/bun": "latest",
60
- "typescript": "^5.8.0"
61
- },
62
- "keywords": [
63
- "outfitter",
64
- "index",
65
- "fts5",
66
- "sqlite",
67
- "search",
68
- "typescript"
69
- ],
70
- "license": "MIT",
71
- "repository": {
72
- "type": "git",
73
- "url": "https://github.com/outfitter-dev/outfitter.git",
74
- "directory": "packages/index"
75
- },
76
- "publishConfig": {
77
- "access": "public"
76
+ "@types/bun": "^1.3.9",
77
+ "typescript": "^5.9.3"
78
78
  }
79
79
  }
@@ -1,46 +0,0 @@
1
- // @bun
2
- // packages/index/src/migrations.ts
3
- import { Result } from "@outfitter/contracts";
4
- function createStorageError(message, cause) {
5
- return {
6
- _tag: "StorageError",
7
- message,
8
- cause
9
- };
10
- }
11
- function createMigrationRegistry() {
12
- const steps = new Map;
13
- return {
14
- register(fromVersion, toVersion, migrate) {
15
- steps.set(fromVersion, { to: toVersion, migrate });
16
- },
17
- migrate(context, fromVersion, toVersion) {
18
- if (fromVersion === toVersion) {
19
- return Result.ok(undefined);
20
- }
21
- let current = fromVersion;
22
- const visited = new Set;
23
- while (current < toVersion) {
24
- if (visited.has(current)) {
25
- return Result.err(createStorageError(`Detected migration loop at version ${current}`));
26
- }
27
- visited.add(current);
28
- const step = steps.get(current);
29
- if (!step) {
30
- return Result.err(createStorageError(`No migration registered from version ${current}`));
31
- }
32
- const result = step.migrate(context);
33
- if (result.isErr()) {
34
- return result;
35
- }
36
- current = step.to;
37
- }
38
- if (current !== toVersion) {
39
- return Result.err(createStorageError(`Migration ended at version ${current} instead of ${toVersion}`));
40
- }
41
- return Result.ok(undefined);
42
- }
43
- };
44
- }
45
-
46
- export { createMigrationRegistry };
@@ -1,251 +0,0 @@
1
- // @bun
2
- import {
3
- INDEX_META_KEY,
4
- INDEX_META_TABLE,
5
- INDEX_VERSION
6
- } from "./index-bbzmc40h.js";
7
-
8
- // packages/index/src/fts5.ts
9
- import { Database } from "bun:sqlite";
10
- import { existsSync, mkdirSync } from "fs";
11
- import { dirname } from "path";
12
- import { Result } from "@outfitter/contracts";
13
- var DEFAULT_TABLE_NAME = "documents";
14
- var DEFAULT_TOKENIZER = "unicode61";
15
- var DEFAULT_LIMIT = 25;
16
- var DEFAULT_OFFSET = 0;
17
- var VALID_TOKENIZERS = {
18
- unicode61: true,
19
- porter: true,
20
- trigram: true
21
- };
22
- var TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
23
- var DEFAULT_TOOL = "outfitter-index";
24
- var DEFAULT_TOOL_VERSION = "0.0.0";
25
- function createStorageError(message, cause) {
26
- return {
27
- _tag: "StorageError",
28
- message,
29
- cause
30
- };
31
- }
32
- function assertValidTableName(tableName) {
33
- if (!TABLE_NAME_PATTERN.test(tableName)) {
34
- throw new Error(`Invalid table name: ${tableName}`);
35
- }
36
- }
37
- function assertValidTokenizer(tokenizer) {
38
- if (!Object.hasOwn(VALID_TOKENIZERS, tokenizer)) {
39
- throw new Error(`Invalid tokenizer: ${tokenizer}`);
40
- }
41
- return tokenizer;
42
- }
43
- function getUserVersion(db) {
44
- const row = db.query("PRAGMA user_version").get();
45
- return row?.user_version ?? 0;
46
- }
47
- function setUserVersion(db, version) {
48
- if (!Number.isInteger(version) || version < 0) {
49
- throw new Error(`Invalid user_version: ${version}`);
50
- }
51
- db.run(`PRAGMA user_version = ${version}`);
52
- }
53
- function ensureMetaTable(db) {
54
- db.run(`CREATE TABLE IF NOT EXISTS ${INDEX_META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
55
- }
56
- function readIndexMetadata(db) {
57
- try {
58
- const row = db.query(`SELECT value FROM ${INDEX_META_TABLE} WHERE key = ?`).get(INDEX_META_KEY);
59
- if (!row) {
60
- return null;
61
- }
62
- const parsed = JSON.parse(row.value);
63
- return parsed;
64
- } catch {
65
- return null;
66
- }
67
- }
68
- function writeIndexMetadata(db, metadata) {
69
- ensureMetaTable(db);
70
- db.run(`INSERT OR REPLACE INTO ${INDEX_META_TABLE} (key, value) VALUES (?, ?)`, [INDEX_META_KEY, JSON.stringify(metadata)]);
71
- }
72
- function createIndex(options) {
73
- const tableName = options.tableName ?? DEFAULT_TABLE_NAME;
74
- assertValidTableName(tableName);
75
- const tokenizer = assertValidTokenizer(options.tokenizer ?? DEFAULT_TOKENIZER);
76
- const tool = options.tool ?? DEFAULT_TOOL;
77
- const toolVersion = options.toolVersion ?? DEFAULT_TOOL_VERSION;
78
- const dir = dirname(options.path);
79
- if (!existsSync(dir)) {
80
- mkdirSync(dir, { recursive: true });
81
- }
82
- const db = new Database(options.path);
83
- db.run("PRAGMA journal_mode=WAL");
84
- const currentVersion = getUserVersion(db);
85
- if (currentVersion === 0) {
86
- const metadata = {
87
- version: INDEX_VERSION,
88
- created: new Date().toISOString(),
89
- tool,
90
- toolVersion
91
- };
92
- setUserVersion(db, INDEX_VERSION);
93
- writeIndexMetadata(db, metadata);
94
- } else if (currentVersion !== INDEX_VERSION) {
95
- if (!options.migrations) {
96
- throw new Error(`Index version ${currentVersion} does not match ${INDEX_VERSION}. Provide migrations or rebuild the index.`);
97
- }
98
- const context = { db };
99
- const result = options.migrations.migrate(context, currentVersion, INDEX_VERSION);
100
- if (result.isErr()) {
101
- throw new Error(`Failed to migrate index: ${result.error.message}`);
102
- }
103
- const existing = readIndexMetadata(db);
104
- const metadata = {
105
- version: INDEX_VERSION,
106
- created: existing?.created ?? new Date().toISOString(),
107
- tool,
108
- toolVersion
109
- };
110
- setUserVersion(db, INDEX_VERSION);
111
- writeIndexMetadata(db, metadata);
112
- } else if (!readIndexMetadata(db)) {
113
- const metadata = {
114
- version: INDEX_VERSION,
115
- created: new Date().toISOString(),
116
- tool,
117
- toolVersion
118
- };
119
- writeIndexMetadata(db, metadata);
120
- }
121
- db.run(`
122
- CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName}
123
- USING fts5(
124
- id UNINDEXED,
125
- content,
126
- metadata UNINDEXED,
127
- tokenize='${tokenizer}'
128
- )
129
- `);
130
- let isClosed = false;
131
- function checkClosed() {
132
- if (isClosed) {
133
- return Result.err(createStorageError("Index is closed"));
134
- }
135
- return Result.ok(undefined);
136
- }
137
- return {
138
- async add(doc) {
139
- const closedCheck = checkClosed();
140
- if (closedCheck.isErr()) {
141
- return closedCheck;
142
- }
143
- try {
144
- const metadataJson = doc.metadata ? JSON.stringify(doc.metadata) : null;
145
- db.run(`DELETE FROM ${tableName} WHERE id = ?`, [doc.id]);
146
- db.run(`INSERT INTO ${tableName} (id, content, metadata) VALUES (?, ?, ?)`, [doc.id, doc.content, metadataJson]);
147
- return Result.ok(undefined);
148
- } catch (error) {
149
- return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to add document", error));
150
- }
151
- },
152
- async addMany(docs) {
153
- const closedCheck = checkClosed();
154
- if (closedCheck.isErr()) {
155
- return closedCheck;
156
- }
157
- try {
158
- db.run("BEGIN TRANSACTION");
159
- try {
160
- const deleteStmt = db.prepare(`DELETE FROM ${tableName} WHERE id = ?`);
161
- const insertStmt = db.prepare(`INSERT INTO ${tableName} (id, content, metadata) VALUES (?, ?, ?)`);
162
- for (const doc of docs) {
163
- const metadataJson = doc.metadata ? JSON.stringify(doc.metadata) : null;
164
- deleteStmt.run(doc.id);
165
- insertStmt.run(doc.id, doc.content, metadataJson);
166
- }
167
- deleteStmt.finalize();
168
- insertStmt.finalize();
169
- db.run("COMMIT");
170
- return Result.ok(undefined);
171
- } catch (error) {
172
- db.run("ROLLBACK");
173
- throw error;
174
- }
175
- } catch (error) {
176
- return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to add documents", error));
177
- }
178
- },
179
- async search(query) {
180
- const closedCheck = checkClosed();
181
- if (closedCheck.isErr()) {
182
- return closedCheck;
183
- }
184
- const limit = query.limit ?? DEFAULT_LIMIT;
185
- const offset = query.offset ?? DEFAULT_OFFSET;
186
- try {
187
- const rows = db.query(`
188
- SELECT
189
- id,
190
- content,
191
- metadata,
192
- bm25(${tableName}) as score,
193
- snippet(${tableName}, 1, '<b>', '</b>', '...', 32) as highlight
194
- FROM ${tableName}
195
- WHERE ${tableName} MATCH ?
196
- ORDER BY bm25(${tableName}) ASC
197
- LIMIT ? OFFSET ?
198
- `).all(query.query, limit, offset);
199
- const results = rows.map((row) => {
200
- const result = {
201
- id: row.id,
202
- content: row.content,
203
- score: row.score,
204
- highlights: [row.highlight]
205
- };
206
- if (row.metadata) {
207
- try {
208
- result.metadata = JSON.parse(row.metadata);
209
- } catch {}
210
- }
211
- return result;
212
- });
213
- return Result.ok(results);
214
- } catch (error) {
215
- return Result.err(createStorageError(error instanceof Error ? error.message : "Search failed", error));
216
- }
217
- },
218
- async remove(id) {
219
- const closedCheck = checkClosed();
220
- if (closedCheck.isErr()) {
221
- return closedCheck;
222
- }
223
- try {
224
- db.run(`DELETE FROM ${tableName} WHERE id = ?`, [id]);
225
- return Result.ok(undefined);
226
- } catch (error) {
227
- return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to remove document", error));
228
- }
229
- },
230
- async clear() {
231
- const closedCheck = checkClosed();
232
- if (closedCheck.isErr()) {
233
- return closedCheck;
234
- }
235
- try {
236
- db.run(`DELETE FROM ${tableName}`);
237
- return Result.ok(undefined);
238
- } catch (error) {
239
- return Result.err(createStorageError(error instanceof Error ? error.message : "Failed to clear index", error));
240
- }
241
- },
242
- close() {
243
- if (!isClosed) {
244
- isClosed = true;
245
- db.close();
246
- }
247
- }
248
- };
249
- }
250
-
251
- export { createIndex };