libsql-search 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.esm.js CHANGED
@@ -786,6 +786,9 @@ function prepareTextForEmbedding(fields) {
786
786
  const SQL_IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
787
787
  const DEFAULT_SEARCH_LIMIT = 10;
788
788
  const MAX_SEARCH_LIMIT = 100;
789
+ const DEFAULT_SEARCH_CANDIDATE_MULTIPLIER = 4;
790
+ const MIN_SEARCH_CANDIDATES = 32;
791
+ const MAX_SEARCH_CANDIDATES = MAX_SEARCH_LIMIT * 10;
789
792
  function validateSqlIdentifier(identifier, name = "identifier") {
790
793
  if (typeof identifier !== "string" || !SQL_IDENTIFIER_PATTERN.test(identifier)) {
791
794
  throw new Error(
@@ -804,6 +807,23 @@ function normalizeSearchLimit(limit = DEFAULT_SEARCH_LIMIT) {
804
807
  }
805
808
  return limit;
806
809
  }
810
+ function defaultSearchCandidates(limit) {
811
+ return Math.min(
812
+ Math.max(limit * DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, MIN_SEARCH_CANDIDATES),
813
+ MAX_SEARCH_CANDIDATES
814
+ );
815
+ }
816
+ function normalizeSearchCandidates(candidates, limit) {
817
+ if (candidates === void 0) {
818
+ return defaultSearchCandidates(limit);
819
+ }
820
+ if (typeof candidates !== "number" || !Number.isFinite(candidates) || !Number.isInteger(candidates) || candidates < limit || candidates > MAX_SEARCH_CANDIDATES) {
821
+ throw new Error(
822
+ `Invalid search candidates: expected an integer from the search limit (${limit}) to ${MAX_SEARCH_CANDIDATES}`
823
+ );
824
+ }
825
+ return candidates;
826
+ }
807
827
  function normalizeVectorDimensions(dimensions) {
808
828
  if (typeof dimensions !== "number" || !Number.isFinite(dimensions) || !Number.isInteger(dimensions) || dimensions < 1) {
809
829
  throw new Error("Invalid vector dimensions: expected a positive integer");
@@ -811,6 +831,16 @@ function normalizeVectorDimensions(dimensions) {
811
831
  return dimensions;
812
832
  }
813
833
 
834
+ class IndexingError extends Error {
835
+ phase;
836
+ failures;
837
+ constructor(message, phase, failures = [], options) {
838
+ super(message, options);
839
+ this.name = "IndexingError";
840
+ this.phase = phase;
841
+ this.failures = failures;
842
+ }
843
+ }
814
844
  async function indexContent(options) {
815
845
  const {
816
846
  client,
@@ -819,32 +849,92 @@ async function indexContent(options) {
819
849
  fileExtensions = [".md", ".markdown"],
820
850
  exclude = ["node_modules", ".git", "dist", "build"],
821
851
  tableName = "articles",
822
- onProgress
852
+ onProgress,
853
+ failurePolicy = "abort",
854
+ allowEmptyIndex = false
823
855
  } = options;
824
856
  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
825
- const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
826
- if (files.length === 0) {
827
- console.warn(`No files found in ${contentPath}`);
828
- return { success: 0, failed: 0, total: 0 };
829
- }
830
- await client.execute(`DELETE FROM ${quotedTableName}`);
831
- let success = 0;
832
- let failed = 0;
857
+ let files;
858
+ try {
859
+ files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
860
+ } catch (error) {
861
+ throw new IndexingError(
862
+ `Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
863
+ "build",
864
+ [],
865
+ { cause: toError(error) }
866
+ );
867
+ }
868
+ files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
869
+ if (files.length === 0 && !allowEmptyIndex) {
870
+ throw new IndexingError(
871
+ `No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
872
+ "build"
873
+ );
874
+ }
875
+ const documents = [];
876
+ const failures = [];
877
+ const slugOwners = /* @__PURE__ */ new Map();
833
878
  for (let i = 0; i < files.length; i++) {
834
879
  const file = files[i];
835
880
  if (onProgress) {
836
881
  onProgress(i + 1, files.length, file.relativePath);
837
882
  }
838
- try {
839
- const document = await processFile(file, embeddingOptions);
840
- await insertDocument(client, document, quotedTableName);
841
- success++;
842
- } catch (error) {
843
- console.error(`Failed to index ${file.relativePath}:`, error);
844
- failed++;
883
+ const outcome = await buildDocument(file, embeddingOptions);
884
+ let failure;
885
+ if (outcome.ok) {
886
+ const owner = slugOwners.get(outcome.document.slug);
887
+ if (owner === void 0) {
888
+ slugOwners.set(outcome.document.slug, file.relativePath);
889
+ documents.push(outcome.document);
890
+ } else {
891
+ failure = {
892
+ file: file.relativePath,
893
+ stage: "parse",
894
+ error: new Error(
895
+ `Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
896
+ )
897
+ };
898
+ }
899
+ } else {
900
+ failure = {
901
+ file: file.relativePath,
902
+ stage: outcome.stage,
903
+ error: outcome.error
904
+ };
905
+ }
906
+ if (failure === void 0) {
907
+ continue;
845
908
  }
909
+ if (failurePolicy === "abort") {
910
+ throw new IndexingError(
911
+ `Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
912
+ "build",
913
+ [failure],
914
+ { cause: failure.error }
915
+ );
916
+ }
917
+ console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
918
+ failures.push(failure);
919
+ }
920
+ if (files.length > 0 && documents.length === 0) {
921
+ throw new IndexingError(
922
+ `All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
923
+ "build",
924
+ failures,
925
+ { cause: failures[0].error }
926
+ );
846
927
  }
847
- return { success, failed, total: files.length };
928
+ const success = documents.length;
929
+ await replaceIndex(client, quotedTableName, documents, failures);
930
+ return {
931
+ success,
932
+ failed: failures.length,
933
+ total: files.length,
934
+ replaced: true,
935
+ partial: failures.length > 0,
936
+ failures
937
+ };
848
938
  }
849
939
  async function findFiles(dir, baseDir, extensions, exclude) {
850
940
  const files = [];
@@ -868,34 +958,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
868
958
  }
869
959
  return files;
870
960
  }
871
- async function processFile(file, embeddingOptions) {
872
- const content = await readFile(file.fullPath, "utf-8");
873
- const { data: frontMatter, content: markdown } = matter(content);
961
+ async function buildDocument(file, embeddingOptions) {
962
+ let raw;
963
+ try {
964
+ raw = await readFile(file.fullPath, "utf-8");
965
+ } catch (error) {
966
+ return { ok: false, stage: "read", error: toError(error) };
967
+ }
968
+ let parsed;
969
+ try {
970
+ parsed = parseFile(file, raw);
971
+ } catch (error) {
972
+ return { ok: false, stage: "parse", error: toError(error) };
973
+ }
974
+ let embedding;
975
+ try {
976
+ embedding = await generateEmbedding(parsed.embeddingText, {
977
+ ...embeddingOptions,
978
+ intent: embeddingOptions.intent ?? "document"
979
+ });
980
+ } catch (error) {
981
+ return { ok: false, stage: "embed", error: toError(error) };
982
+ }
983
+ return {
984
+ ok: true,
985
+ document: {
986
+ slug: parsed.slug,
987
+ title: parsed.title,
988
+ content: parsed.content,
989
+ folder: file.folder,
990
+ tags: parsed.tags,
991
+ serializedTags: parsed.serializedTags,
992
+ embedding,
993
+ metadata: parsed.metadata
994
+ }
995
+ };
996
+ }
997
+ function parseFile(file, raw) {
998
+ const { data: frontMatter, content: markdown } = matter(raw);
874
999
  const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
875
- const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
1000
+ const title = resolveTitle(file, frontMatter.title);
876
1001
  const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
1002
+ const serializedTags = JSON.stringify(tags);
877
1003
  const embeddingText = prepareTextForEmbedding({
878
1004
  title,
879
1005
  description: frontMatter.description,
880
1006
  content: markdown,
881
1007
  tags
882
1008
  });
883
- const embedding = await generateEmbedding(embeddingText, {
884
- ...embeddingOptions,
885
- intent: embeddingOptions.intent ?? "document"
886
- });
887
1009
  return {
888
1010
  slug,
889
1011
  title,
890
1012
  content: markdown,
891
- folder: file.folder,
892
1013
  tags,
893
- embedding,
1014
+ serializedTags,
1015
+ embeddingText,
894
1016
  metadata: frontMatter
895
1017
  };
896
1018
  }
897
- async function insertDocument(client, document, quotedTableName) {
898
- await client.execute({
1019
+ function resolveTitle(file, rawTitle) {
1020
+ if (!rawTitle) {
1021
+ return fallbackTitle(file);
1022
+ }
1023
+ if (typeof rawTitle === "string") {
1024
+ return rawTitle;
1025
+ }
1026
+ if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
1027
+ return String(rawTitle);
1028
+ }
1029
+ if (rawTitle instanceof Date) {
1030
+ return rawTitle.toISOString();
1031
+ }
1032
+ throw new Error(
1033
+ `Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
1034
+ );
1035
+ }
1036
+ function fallbackTitle(file) {
1037
+ return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
1038
+ }
1039
+ function describeType(value) {
1040
+ return Array.isArray(value) ? "array" : typeof value;
1041
+ }
1042
+ async function replaceIndex(client, quotedTableName, documents, failures) {
1043
+ const statements = [`DELETE FROM ${quotedTableName}`];
1044
+ try {
1045
+ for (let i = 0; i < documents.length; i++) {
1046
+ const document = documents[i];
1047
+ if (document !== void 0) {
1048
+ statements.push(createInsertStatement(document, quotedTableName));
1049
+ documents[i] = void 0;
1050
+ }
1051
+ }
1052
+ } finally {
1053
+ documents.length = 0;
1054
+ }
1055
+ try {
1056
+ await client.batch(statements, "write");
1057
+ } catch (error) {
1058
+ throw new IndexingError(
1059
+ `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
1060
+ "replace",
1061
+ failures,
1062
+ { cause: toError(error) }
1063
+ );
1064
+ }
1065
+ }
1066
+ function createInsertStatement(document, quotedTableName) {
1067
+ return {
899
1068
  sql: `INSERT INTO ${quotedTableName}
900
1069
  (slug, title, content, folder, tags, embedding, created_at, updated_at)
901
1070
  VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
@@ -904,10 +1073,13 @@ async function insertDocument(client, document, quotedTableName) {
904
1073
  document.title,
905
1074
  document.content,
906
1075
  document.folder,
907
- JSON.stringify(document.tags),
1076
+ document.serializedTags,
908
1077
  JSON.stringify(document.embedding)
909
1078
  ]
910
- });
1079
+ };
1080
+ }
1081
+ function toError(error) {
1082
+ return error instanceof Error ? error : new Error(String(error));
911
1083
  }
912
1084
  async function createTable(client, tableName = "articles", dimensions = 384) {
913
1085
  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -942,39 +1114,102 @@ async function createTable(client, tableName = "articles", dimensions = 384) {
942
1114
  `);
943
1115
  }
944
1116
 
1117
+ const RESULT_COLUMNS = "a.id, a.slug, a.title, a.content, a.folder, a.tags, a.created_at";
1118
+ const MISSING_VECTOR_INDEX_PATTERNS = [
1119
+ /failed to parse vector index parameters/i
1120
+ ];
1121
+ const NO_VECTOR_SUPPORT_PATTERNS = [
1122
+ /no such table:\s*vector_top_k/i
1123
+ ];
945
1124
  async function search(options) {
946
1125
  const {
947
1126
  client,
948
1127
  query,
949
1128
  limit = 10,
950
1129
  tableName = "articles",
951
- embeddingOptions = {}
1130
+ embeddingOptions = {},
1131
+ candidates,
1132
+ exact = false
952
1133
  } = options;
953
1134
  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
954
1135
  const resultLimit = normalizeSearchLimit(limit);
1136
+ const embeddingIndexName = validateSqlIdentifier(
1137
+ `${tableName}_embedding_idx`,
1138
+ "embedding index name"
1139
+ );
1140
+ const candidateCount = normalizeSearchCandidates(candidates, resultLimit);
955
1141
  const queryEmbedding = await generateEmbedding(query, {
956
1142
  ...embeddingOptions,
957
1143
  intent: embeddingOptions.intent ?? "query"
958
1144
  });
959
- const results = await client.execute({
1145
+ const queryVector = JSON.stringify(queryEmbedding);
1146
+ const results = exact ? await executeExactSearch(client, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
1147
+ client,
1148
+ quotedTableName,
1149
+ embeddingIndexName,
1150
+ queryVector,
1151
+ candidateCount,
1152
+ resultLimit
1153
+ );
1154
+ return results.rows.map(toSearchResult);
1155
+ }
1156
+ async function executeIndexSearch(client, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
1157
+ try {
1158
+ return await client.execute({
1159
+ sql: `
1160
+ SELECT
1161
+ ${RESULT_COLUMNS},
1162
+ vector_distance_cos(a.embedding, vector(:queryVector)) as distance
1163
+ FROM vector_top_k(:indexName, vector(:queryVector), :candidates) AS v
1164
+ JOIN ${quotedTableName} a ON a.rowid = v.id
1165
+ ORDER BY distance, a.id
1166
+ LIMIT :resultLimit
1167
+ `,
1168
+ args: {
1169
+ queryVector,
1170
+ indexName: embeddingIndexName,
1171
+ candidates: candidateCount,
1172
+ resultLimit
1173
+ }
1174
+ });
1175
+ } catch (error) {
1176
+ throw wrapIndexPathError(error, embeddingIndexName);
1177
+ }
1178
+ }
1179
+ async function executeExactSearch(client, quotedTableName, queryVector, resultLimit) {
1180
+ return client.execute({
960
1181
  sql: `
961
1182
  SELECT
962
- id,
963
- slug,
964
- title,
965
- content,
966
- folder,
967
- tags,
968
- created_at,
969
- vector_distance_cos(embedding, vector(?)) as distance
970
- FROM ${quotedTableName}
971
- WHERE embedding IS NOT NULL
972
- ORDER BY distance
973
- LIMIT ?
1183
+ ${RESULT_COLUMNS},
1184
+ vector_distance_cos(a.embedding, vector(:queryVector)) as distance
1185
+ FROM ${quotedTableName} a
1186
+ WHERE a.embedding IS NOT NULL
1187
+ ORDER BY distance, a.id
1188
+ LIMIT :resultLimit
974
1189
  `,
975
- args: [JSON.stringify(queryEmbedding), resultLimit]
1190
+ args: { queryVector, resultLimit }
976
1191
  });
977
- return results.rows.map((row) => ({
1192
+ }
1193
+ function wrapIndexPathError(error, embeddingIndexName) {
1194
+ if (!(error instanceof Error)) {
1195
+ return error;
1196
+ }
1197
+ if (MISSING_VECTOR_INDEX_PATTERNS.some((pattern) => pattern.test(error.message))) {
1198
+ return new Error(
1199
+ `Vector index "${embeddingIndexName}" could not be used for search. createTable() creates this index; a table created before it existed, or created by hand, will not have it. Create it with CREATE INDEX IF NOT EXISTS "${embeddingIndexName}" ON <table>(libsql_vector_idx(embedding)), or pass exact: true to search without the index.`,
1200
+ { cause: error }
1201
+ );
1202
+ }
1203
+ if (NO_VECTOR_SUPPORT_PATTERNS.some((pattern) => pattern.test(error.message))) {
1204
+ return new Error(
1205
+ `This libSQL deployment has no vector index support: vector_top_k() is unavailable. Pass exact: true to search on the full-scan path.`,
1206
+ { cause: error }
1207
+ );
1208
+ }
1209
+ return error;
1210
+ }
1211
+ function toSearchResult(row) {
1212
+ return {
978
1213
  id: row.id,
979
1214
  slug: row.slug,
980
1215
  title: row.title,
@@ -983,7 +1218,7 @@ async function search(options) {
983
1218
  tags: JSON.parse(row.tags || "[]"),
984
1219
  distance: row.distance,
985
1220
  created_at: row.created_at
986
- }));
1221
+ };
987
1222
  }
988
1223
  async function getAllArticles(client, tableName = "articles") {
989
1224
  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -1057,4 +1292,4 @@ async function getFolders(client, tableName = "articles") {
1057
1292
  return results.rows.map((row) => row.folder);
1058
1293
  }
1059
1294
 
1060
- export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
1295
+ export { DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, IndexingError, MAX_SEARCH_CANDIDATES, MIN_SEARCH_CANDIDATES, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
package/docs/API.md CHANGED
@@ -18,6 +18,10 @@
18
18
  - `validateEmbeddingBatch`
19
19
  - `padEmbedding`
20
20
  - `prepareTextForEmbedding`
21
+ - `IndexingError`
22
+ - `DEFAULT_SEARCH_CANDIDATE_MULTIPLIER`
23
+ - `MIN_SEARCH_CANDIDATES`
24
+ - `MAX_SEARCH_CANDIDATES`
21
25
 
22
26
  It also exports these types:
23
27
 
@@ -34,6 +38,11 @@ It also exports these types:
34
38
  - `EmbeddingOptions`
35
39
  - `IndexerOptions`
36
40
  - `IndexedDocument`
41
+ - `IndexResult`
42
+ - `IndexFailure`
43
+ - `IndexFailurePolicy`
44
+ - `IndexFailureStage`
45
+ - `IndexingErrorPhase`
37
46
  - `SearchOptions`
38
47
  - `SearchResult`
39
48
 
@@ -86,6 +95,8 @@ interface IndexerOptions {
86
95
  exclude?: string[];
87
96
  tableName?: string;
88
97
  onProgress?: (current: number, total: number, file: string) => void;
98
+ failurePolicy?: "abort" | "skip";
99
+ allowEmptyIndex?: boolean;
89
100
  }
90
101
  ```
91
102
 
@@ -94,25 +105,80 @@ Defaults:
94
105
  - `fileExtensions`: [".md", ".markdown"]
95
106
  - `exclude`: ["node_modules", ".git", "dist", "build"]
96
107
  - `tableName`: `"articles"`
108
+ - `failurePolicy`: `"abort"`
109
+ - `allowEmptyIndex`: `false`
97
110
 
98
111
  Return shape:
99
112
 
100
113
  ```ts
101
- {
102
- success: number;
103
- failed: number;
104
- total: number;
114
+ interface IndexResult {
115
+ success: number; // documents written
116
+ failed: number; // files that could not be indexed
117
+ total: number; // files discovered on disk
118
+ replaced: boolean; // whether table contents were replaced by this call
119
+ partial: boolean; // replaced, but some files were skipped
120
+ failures: IndexFailure[];
121
+ }
122
+
123
+ interface IndexFailure {
124
+ file: string; // path relative to contentPath
125
+ stage: "read" | "parse" | "embed";
126
+ error: Error;
105
127
  }
106
128
  ```
107
129
 
108
130
  Behavior notes:
109
131
 
110
- - `indexContent()` deletes existing rows in the target table before rebuilding
111
- - rebuilds are not transactional
132
+ - every file is read, parsed, and embedded in memory before any database state changes
133
+ - the target table is then replaced in a single write transaction, so a failed rebuild leaves the previous index exactly as it was
134
+ - that costs peak memory proportional to the whole corpus, and against remote clients the replacement travels as a single un-chunked batch request; see [Costs of the two-phase rebuild](./INDEXING.md#costs-of-the-two-phase-rebuild) before rebuilding a very large corpus in place
135
+ - files are discovered and indexed in sorted path order
136
+ - frontmatter `title` must be a scalar; a structured title such as a YAML list fails the file at the `parse` stage
137
+ - two files that reduce to the same slug (`foo.md` and `foo.markdown`) collide: the first in sorted path order keeps the slug and the later file is reported as a `parse` failure
138
+ - `failurePolicy: "abort"` throws `IndexingError` on the first file that fails
139
+ - `failurePolicy: "skip"` drops the failing file, records it in `failures`, and rebuilds from the survivors, returning `partial: true`
140
+ - under `"skip"`, if every discovered file fails, the rebuild throws instead of replacing a valid index with an empty one
141
+ - an empty source directory throws unless `allowEmptyIndex: true`, which intentionally empties the index
142
+ - `onProgress` is called once per file during the build phase
112
143
  - frontmatter `title`, `description`, and `tags` are folded into the embedding text
113
144
  - embeddings default to `intent: "document"` unless `embeddingOptions.intent` is set explicitly
114
145
  - if a file has no frontmatter title, the filename becomes the title
115
146
 
147
+ ### `IndexingError`
148
+
149
+ Thrown when a rebuild cannot complete. The previously indexed rows are always left unchanged.
150
+
151
+ ```ts
152
+ class IndexingError extends Error {
153
+ readonly phase: "build" | "replace";
154
+ readonly failures: IndexFailure[];
155
+ }
156
+ ```
157
+
158
+ - `phase: "build"` means the failure happened before any database work: a file failed, every file failed, the source directory was empty, or it could not be scanned
159
+ - `phase: "replace"` means the replacement transaction failed and was rolled back
160
+ - `cause` carries the underlying error
161
+ - on a `phase: "replace"` error, `failures` lists files skipped during the build phase. They are not the cause of the rollback, which is carried by `cause`
162
+
163
+ ```ts
164
+ import { indexContent, IndexingError } from "libsql-search";
165
+
166
+ try {
167
+ await indexContent({ client, contentPath: "./content" });
168
+ } catch (error) {
169
+ if (error instanceof IndexingError) {
170
+ console.error(error.phase, error.failures);
171
+ }
172
+
173
+ throw error;
174
+ }
175
+ ```
176
+
177
+ Breaking changes in this behavior:
178
+
179
+ - partial failures previously counted into `failed` and still replaced the table; they now throw. Pass `failurePolicy: "skip"` for the previous lenient behavior.
180
+ - an empty source directory previously returned zeros and left stale rows in place; it now throws. Pass `allowEmptyIndex: true` to intentionally empty the index.
181
+
116
182
  ## `search(options)`
117
183
 
118
184
  Generates a query embedding and performs vector similarity search.
@@ -124,6 +190,8 @@ interface SearchOptions {
124
190
  limit?: number;
125
191
  tableName?: string;
126
192
  embeddingOptions?: EmbeddingOptions;
193
+ candidates?: number;
194
+ exact?: boolean;
127
195
  }
128
196
  ```
129
197
 
@@ -131,9 +199,65 @@ Defaults:
131
199
 
132
200
  - `limit`: `10`
133
201
  - `tableName`: `"articles"`
202
+ - `candidates`: `Math.max(limit * 4, 32)`
203
+ - `exact`: `false`
134
204
 
135
205
  `limit` must be an integer from `1` through `100`; invalid values are rejected before query embedding generation.
136
206
 
207
+ ### Search Is Approximate By Default
208
+
209
+ The default path queries the `<tableName>_embedding_idx` vector index through libSQL's `vector_top_k()`. **That index is an approximate-nearest-neighbor structure. It can miss a true nearest neighbor.** There is no configuration that makes the index path exact; only `exact: true` guarantees exactness.
210
+
211
+ To limit the accuracy loss, the default path over-fetches: it pulls `candidates` rows from the index, recomputes the true `vector_distance_cos` for each, and orders exactly by `(distance, id)` before trimming to `limit`. So:
212
+
213
+ - **Recall is approximate.** A row the index does not return as a candidate cannot appear in the results, no matter its true distance. Raising `candidates` raises recall.
214
+ - **Ranking within the candidate set is exact.** Distances on returned rows are always the true cosine distances, not index approximations.
215
+ - **Ordering is fully deterministic.** The `id` tiebreaker means two rows at an identical distance always come back in the same order, even though the index's own candidate order is not stable across runs.
216
+ - **Rows with a `NULL` embedding are never returned.** The vector index excludes them on its own.
217
+
218
+ **Both paths order by `(distance, id)`.** The determinism guarantee covers `exact: true` as well as the default, so the two paths return identical orderings for identical inputs and can be compared directly. This is a change in tie ordering for the exact path, which previously sorted by distance alone and left tied rows in whatever order the scan produced.
219
+
220
+ ### `candidates`
221
+
222
+ Controls how many rows the index returns for the exact re-rank. It must be an integer from `limit` through `MAX_SEARCH_CANDIDATES`; a value below `limit` is rejected rather than clamped, because it would silently truncate the result set. Invalid values throw before query embedding generation and before any database call.
223
+
224
+ ```ts
225
+ // Trade query cost for recall on a large corpus
226
+ const results = await search({ client, query, limit: 10, candidates: 200 });
227
+ ```
228
+
229
+ `candidates` has no effect when `exact` is `true` — that path scans every row — but it is **still validated**. `search({ exact: true, limit: 10, candidates: 5 })` throws, exactly as it would on the index path. Validity does not depend on which path a call happens to take.
230
+
231
+ Related exported constants:
232
+
233
+ - `DEFAULT_SEARCH_CANDIDATE_MULTIPLIER` (`4`): multiplier applied to `limit`
234
+ - `MIN_SEARCH_CANDIDATES` (`32`): floor for the derived default
235
+ - `MAX_SEARCH_CANDIDATES` (`1000`): ceiling for any explicit value. The derived default cannot reach it today, since `limit` tops out at `100`; the cap constrains explicit values.
236
+
237
+ ### `exact`
238
+
239
+ Set `exact: true` to bypass the index and score every row in the table.
240
+
241
+ ```ts
242
+ const results = await search({ client, query, exact: true });
243
+ ```
244
+
245
+ This is the guaranteed-exact path: it computes `vector_distance_cos` for every row with a non-`NULL` embedding, sorts by `(distance, id)`, and trims to `limit`. Cost grows linearly with table size, so it is intended for small corpora, correctness checks against the index path, and tables that have no vector index.
246
+
247
+ ### Missing Vector Index
248
+
249
+ If the target table has no `<tableName>_embedding_idx`, the default path throws an error naming the missing index and pointing at `createTable()` and `exact: true`. libSQL's own message for this case ("failed to parse vector index parameters") says nothing about a missing index, so it is preserved as the thrown error's `cause` rather than surfaced directly.
250
+
251
+ A deployment with no vector support at all is a separate case with its own message. `vector_top_k()` does not exist there, so no `CREATE INDEX` can help and the error points only at `exact: true`.
252
+
253
+ **Those two messages are the only ones rewritten.** Every other failure from the index path reaches the caller with its original message intact. In particular, a width mismatch between the query embedding and the `embedding` column surfaces as libSQL's own `vector index(search): dimensions are different: 384 != 4`, which names both widths and is the useful diagnostic for the dimension drift described in the [Migration and reindexing guide](./MIGRATIONS.md).
254
+
255
+ Search never falls back to the exact scan on its own. A silent fallback would turn a one-line schema fix into an invisible, permanent full-table scan.
256
+
257
+ ### Requirements
258
+
259
+ `vector_top_k()` and `libsql_vector_idx()` require a libSQL build with native vector support. The peer dependency is `@libsql/client ^0.15.0`; this behavior is verified against `@libsql/client` `0.15.15` with a local `:memory:` database. Remote Turso/libSQL servers must also support vector indexes — that is a property of the server, not the client, and no minimum server version is claimed here beyond that requirement. A deployment without it fails the default path with an error saying so and naming `exact: true` as the remedy. Use `exact: true` against any deployment where vector index support is unavailable or unverified.
260
+
137
261
  Result shape:
138
262
 
139
263
  ```ts