libsql-search 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/index.cjs +287 -48
- package/dist/index.d.ts +104 -9
- package/dist/index.esm.js +284 -49
- package/docs/API.md +130 -6
- package/docs/INDEXING.md +114 -4
- package/docs/MIGRATIONS.md +23 -1
- package/docs/RELEASING.md +11 -3
- package/docs/TROUBLESHOOTING.md +21 -0
- package/package.json +1 -1
package/dist/index.esm.js
CHANGED
|
@@ -786,6 +786,9 @@ function prepareTextForEmbedding(fields) {
|
|
|
786
786
|
const SQL_IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
787
787
|
const DEFAULT_SEARCH_LIMIT = 10;
|
|
788
788
|
const MAX_SEARCH_LIMIT = 100;
|
|
789
|
+
const DEFAULT_SEARCH_CANDIDATE_MULTIPLIER = 4;
|
|
790
|
+
const MIN_SEARCH_CANDIDATES = 32;
|
|
791
|
+
const MAX_SEARCH_CANDIDATES = MAX_SEARCH_LIMIT * 10;
|
|
789
792
|
function validateSqlIdentifier(identifier, name = "identifier") {
|
|
790
793
|
if (typeof identifier !== "string" || !SQL_IDENTIFIER_PATTERN.test(identifier)) {
|
|
791
794
|
throw new Error(
|
|
@@ -804,6 +807,23 @@ function normalizeSearchLimit(limit = DEFAULT_SEARCH_LIMIT) {
|
|
|
804
807
|
}
|
|
805
808
|
return limit;
|
|
806
809
|
}
|
|
810
|
+
function defaultSearchCandidates(limit) {
|
|
811
|
+
return Math.min(
|
|
812
|
+
Math.max(limit * DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, MIN_SEARCH_CANDIDATES),
|
|
813
|
+
MAX_SEARCH_CANDIDATES
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
function normalizeSearchCandidates(candidates, limit) {
|
|
817
|
+
if (candidates === void 0) {
|
|
818
|
+
return defaultSearchCandidates(limit);
|
|
819
|
+
}
|
|
820
|
+
if (typeof candidates !== "number" || !Number.isFinite(candidates) || !Number.isInteger(candidates) || candidates < limit || candidates > MAX_SEARCH_CANDIDATES) {
|
|
821
|
+
throw new Error(
|
|
822
|
+
`Invalid search candidates: expected an integer from the search limit (${limit}) to ${MAX_SEARCH_CANDIDATES}`
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
return candidates;
|
|
826
|
+
}
|
|
807
827
|
function normalizeVectorDimensions(dimensions) {
|
|
808
828
|
if (typeof dimensions !== "number" || !Number.isFinite(dimensions) || !Number.isInteger(dimensions) || dimensions < 1) {
|
|
809
829
|
throw new Error("Invalid vector dimensions: expected a positive integer");
|
|
@@ -811,6 +831,16 @@ function normalizeVectorDimensions(dimensions) {
|
|
|
811
831
|
return dimensions;
|
|
812
832
|
}
|
|
813
833
|
|
|
834
|
+
class IndexingError extends Error {
|
|
835
|
+
phase;
|
|
836
|
+
failures;
|
|
837
|
+
constructor(message, phase, failures = [], options) {
|
|
838
|
+
super(message, options);
|
|
839
|
+
this.name = "IndexingError";
|
|
840
|
+
this.phase = phase;
|
|
841
|
+
this.failures = failures;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
814
844
|
async function indexContent(options) {
|
|
815
845
|
const {
|
|
816
846
|
client,
|
|
@@ -819,32 +849,92 @@ async function indexContent(options) {
|
|
|
819
849
|
fileExtensions = [".md", ".markdown"],
|
|
820
850
|
exclude = ["node_modules", ".git", "dist", "build"],
|
|
821
851
|
tableName = "articles",
|
|
822
|
-
onProgress
|
|
852
|
+
onProgress,
|
|
853
|
+
failurePolicy = "abort",
|
|
854
|
+
allowEmptyIndex = false
|
|
823
855
|
} = options;
|
|
824
856
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
857
|
+
let files;
|
|
858
|
+
try {
|
|
859
|
+
files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
|
|
860
|
+
} catch (error) {
|
|
861
|
+
throw new IndexingError(
|
|
862
|
+
`Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
|
|
863
|
+
"build",
|
|
864
|
+
[],
|
|
865
|
+
{ cause: toError(error) }
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
|
|
869
|
+
if (files.length === 0 && !allowEmptyIndex) {
|
|
870
|
+
throw new IndexingError(
|
|
871
|
+
`No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
|
|
872
|
+
"build"
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
const documents = [];
|
|
876
|
+
const failures = [];
|
|
877
|
+
const slugOwners = /* @__PURE__ */ new Map();
|
|
833
878
|
for (let i = 0; i < files.length; i++) {
|
|
834
879
|
const file = files[i];
|
|
835
880
|
if (onProgress) {
|
|
836
881
|
onProgress(i + 1, files.length, file.relativePath);
|
|
837
882
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
883
|
+
const outcome = await buildDocument(file, embeddingOptions);
|
|
884
|
+
let failure;
|
|
885
|
+
if (outcome.ok) {
|
|
886
|
+
const owner = slugOwners.get(outcome.document.slug);
|
|
887
|
+
if (owner === void 0) {
|
|
888
|
+
slugOwners.set(outcome.document.slug, file.relativePath);
|
|
889
|
+
documents.push(outcome.document);
|
|
890
|
+
} else {
|
|
891
|
+
failure = {
|
|
892
|
+
file: file.relativePath,
|
|
893
|
+
stage: "parse",
|
|
894
|
+
error: new Error(
|
|
895
|
+
`Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
|
|
896
|
+
)
|
|
897
|
+
};
|
|
898
|
+
}
|
|
899
|
+
} else {
|
|
900
|
+
failure = {
|
|
901
|
+
file: file.relativePath,
|
|
902
|
+
stage: outcome.stage,
|
|
903
|
+
error: outcome.error
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
if (failure === void 0) {
|
|
907
|
+
continue;
|
|
845
908
|
}
|
|
909
|
+
if (failurePolicy === "abort") {
|
|
910
|
+
throw new IndexingError(
|
|
911
|
+
`Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
|
|
912
|
+
"build",
|
|
913
|
+
[failure],
|
|
914
|
+
{ cause: failure.error }
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
|
|
918
|
+
failures.push(failure);
|
|
919
|
+
}
|
|
920
|
+
if (files.length > 0 && documents.length === 0) {
|
|
921
|
+
throw new IndexingError(
|
|
922
|
+
`All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
|
|
923
|
+
"build",
|
|
924
|
+
failures,
|
|
925
|
+
{ cause: failures[0].error }
|
|
926
|
+
);
|
|
846
927
|
}
|
|
847
|
-
|
|
928
|
+
const success = documents.length;
|
|
929
|
+
await replaceIndex(client, quotedTableName, documents, failures);
|
|
930
|
+
return {
|
|
931
|
+
success,
|
|
932
|
+
failed: failures.length,
|
|
933
|
+
total: files.length,
|
|
934
|
+
replaced: true,
|
|
935
|
+
partial: failures.length > 0,
|
|
936
|
+
failures
|
|
937
|
+
};
|
|
848
938
|
}
|
|
849
939
|
async function findFiles(dir, baseDir, extensions, exclude) {
|
|
850
940
|
const files = [];
|
|
@@ -868,34 +958,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
|
|
|
868
958
|
}
|
|
869
959
|
return files;
|
|
870
960
|
}
|
|
871
|
-
async function
|
|
872
|
-
|
|
873
|
-
|
|
961
|
+
async function buildDocument(file, embeddingOptions) {
|
|
962
|
+
let raw;
|
|
963
|
+
try {
|
|
964
|
+
raw = await readFile(file.fullPath, "utf-8");
|
|
965
|
+
} catch (error) {
|
|
966
|
+
return { ok: false, stage: "read", error: toError(error) };
|
|
967
|
+
}
|
|
968
|
+
let parsed;
|
|
969
|
+
try {
|
|
970
|
+
parsed = parseFile(file, raw);
|
|
971
|
+
} catch (error) {
|
|
972
|
+
return { ok: false, stage: "parse", error: toError(error) };
|
|
973
|
+
}
|
|
974
|
+
let embedding;
|
|
975
|
+
try {
|
|
976
|
+
embedding = await generateEmbedding(parsed.embeddingText, {
|
|
977
|
+
...embeddingOptions,
|
|
978
|
+
intent: embeddingOptions.intent ?? "document"
|
|
979
|
+
});
|
|
980
|
+
} catch (error) {
|
|
981
|
+
return { ok: false, stage: "embed", error: toError(error) };
|
|
982
|
+
}
|
|
983
|
+
return {
|
|
984
|
+
ok: true,
|
|
985
|
+
document: {
|
|
986
|
+
slug: parsed.slug,
|
|
987
|
+
title: parsed.title,
|
|
988
|
+
content: parsed.content,
|
|
989
|
+
folder: file.folder,
|
|
990
|
+
tags: parsed.tags,
|
|
991
|
+
serializedTags: parsed.serializedTags,
|
|
992
|
+
embedding,
|
|
993
|
+
metadata: parsed.metadata
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
function parseFile(file, raw) {
|
|
998
|
+
const { data: frontMatter, content: markdown } = matter(raw);
|
|
874
999
|
const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
|
|
875
|
-
const title = frontMatter.title
|
|
1000
|
+
const title = resolveTitle(file, frontMatter.title);
|
|
876
1001
|
const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
|
|
1002
|
+
const serializedTags = JSON.stringify(tags);
|
|
877
1003
|
const embeddingText = prepareTextForEmbedding({
|
|
878
1004
|
title,
|
|
879
1005
|
description: frontMatter.description,
|
|
880
1006
|
content: markdown,
|
|
881
1007
|
tags
|
|
882
1008
|
});
|
|
883
|
-
const embedding = await generateEmbedding(embeddingText, {
|
|
884
|
-
...embeddingOptions,
|
|
885
|
-
intent: embeddingOptions.intent ?? "document"
|
|
886
|
-
});
|
|
887
1009
|
return {
|
|
888
1010
|
slug,
|
|
889
1011
|
title,
|
|
890
1012
|
content: markdown,
|
|
891
|
-
folder: file.folder,
|
|
892
1013
|
tags,
|
|
893
|
-
|
|
1014
|
+
serializedTags,
|
|
1015
|
+
embeddingText,
|
|
894
1016
|
metadata: frontMatter
|
|
895
1017
|
};
|
|
896
1018
|
}
|
|
897
|
-
|
|
898
|
-
|
|
1019
|
+
function resolveTitle(file, rawTitle) {
|
|
1020
|
+
if (!rawTitle) {
|
|
1021
|
+
return fallbackTitle(file);
|
|
1022
|
+
}
|
|
1023
|
+
if (typeof rawTitle === "string") {
|
|
1024
|
+
return rawTitle;
|
|
1025
|
+
}
|
|
1026
|
+
if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
|
|
1027
|
+
return String(rawTitle);
|
|
1028
|
+
}
|
|
1029
|
+
if (rawTitle instanceof Date) {
|
|
1030
|
+
return rawTitle.toISOString();
|
|
1031
|
+
}
|
|
1032
|
+
throw new Error(
|
|
1033
|
+
`Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
function fallbackTitle(file) {
|
|
1037
|
+
return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
|
|
1038
|
+
}
|
|
1039
|
+
function describeType(value) {
|
|
1040
|
+
return Array.isArray(value) ? "array" : typeof value;
|
|
1041
|
+
}
|
|
1042
|
+
async function replaceIndex(client, quotedTableName, documents, failures) {
|
|
1043
|
+
const statements = [`DELETE FROM ${quotedTableName}`];
|
|
1044
|
+
try {
|
|
1045
|
+
for (let i = 0; i < documents.length; i++) {
|
|
1046
|
+
const document = documents[i];
|
|
1047
|
+
if (document !== void 0) {
|
|
1048
|
+
statements.push(createInsertStatement(document, quotedTableName));
|
|
1049
|
+
documents[i] = void 0;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
} finally {
|
|
1053
|
+
documents.length = 0;
|
|
1054
|
+
}
|
|
1055
|
+
try {
|
|
1056
|
+
await client.batch(statements, "write");
|
|
1057
|
+
} catch (error) {
|
|
1058
|
+
throw new IndexingError(
|
|
1059
|
+
`Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
|
|
1060
|
+
"replace",
|
|
1061
|
+
failures,
|
|
1062
|
+
{ cause: toError(error) }
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
function createInsertStatement(document, quotedTableName) {
|
|
1067
|
+
return {
|
|
899
1068
|
sql: `INSERT INTO ${quotedTableName}
|
|
900
1069
|
(slug, title, content, folder, tags, embedding, created_at, updated_at)
|
|
901
1070
|
VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
|
|
@@ -904,10 +1073,13 @@ async function insertDocument(client, document, quotedTableName) {
|
|
|
904
1073
|
document.title,
|
|
905
1074
|
document.content,
|
|
906
1075
|
document.folder,
|
|
907
|
-
|
|
1076
|
+
document.serializedTags,
|
|
908
1077
|
JSON.stringify(document.embedding)
|
|
909
1078
|
]
|
|
910
|
-
}
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
function toError(error) {
|
|
1082
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
911
1083
|
}
|
|
912
1084
|
async function createTable(client, tableName = "articles", dimensions = 384) {
|
|
913
1085
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
@@ -942,39 +1114,102 @@ async function createTable(client, tableName = "articles", dimensions = 384) {
|
|
|
942
1114
|
`);
|
|
943
1115
|
}
|
|
944
1116
|
|
|
1117
|
+
const RESULT_COLUMNS = "a.id, a.slug, a.title, a.content, a.folder, a.tags, a.created_at";
|
|
1118
|
+
const MISSING_VECTOR_INDEX_PATTERNS = [
|
|
1119
|
+
/failed to parse vector index parameters/i
|
|
1120
|
+
];
|
|
1121
|
+
const NO_VECTOR_SUPPORT_PATTERNS = [
|
|
1122
|
+
/no such table:\s*vector_top_k/i
|
|
1123
|
+
];
|
|
945
1124
|
async function search(options) {
|
|
946
1125
|
const {
|
|
947
1126
|
client,
|
|
948
1127
|
query,
|
|
949
1128
|
limit = 10,
|
|
950
1129
|
tableName = "articles",
|
|
951
|
-
embeddingOptions = {}
|
|
1130
|
+
embeddingOptions = {},
|
|
1131
|
+
candidates,
|
|
1132
|
+
exact = false
|
|
952
1133
|
} = options;
|
|
953
1134
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
954
1135
|
const resultLimit = normalizeSearchLimit(limit);
|
|
1136
|
+
const embeddingIndexName = validateSqlIdentifier(
|
|
1137
|
+
`${tableName}_embedding_idx`,
|
|
1138
|
+
"embedding index name"
|
|
1139
|
+
);
|
|
1140
|
+
const candidateCount = normalizeSearchCandidates(candidates, resultLimit);
|
|
955
1141
|
const queryEmbedding = await generateEmbedding(query, {
|
|
956
1142
|
...embeddingOptions,
|
|
957
1143
|
intent: embeddingOptions.intent ?? "query"
|
|
958
1144
|
});
|
|
959
|
-
const
|
|
1145
|
+
const queryVector = JSON.stringify(queryEmbedding);
|
|
1146
|
+
const results = exact ? await executeExactSearch(client, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
|
|
1147
|
+
client,
|
|
1148
|
+
quotedTableName,
|
|
1149
|
+
embeddingIndexName,
|
|
1150
|
+
queryVector,
|
|
1151
|
+
candidateCount,
|
|
1152
|
+
resultLimit
|
|
1153
|
+
);
|
|
1154
|
+
return results.rows.map(toSearchResult);
|
|
1155
|
+
}
|
|
1156
|
+
async function executeIndexSearch(client, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
|
|
1157
|
+
try {
|
|
1158
|
+
return await client.execute({
|
|
1159
|
+
sql: `
|
|
1160
|
+
SELECT
|
|
1161
|
+
${RESULT_COLUMNS},
|
|
1162
|
+
vector_distance_cos(a.embedding, vector(:queryVector)) as distance
|
|
1163
|
+
FROM vector_top_k(:indexName, vector(:queryVector), :candidates) AS v
|
|
1164
|
+
JOIN ${quotedTableName} a ON a.rowid = v.id
|
|
1165
|
+
ORDER BY distance, a.id
|
|
1166
|
+
LIMIT :resultLimit
|
|
1167
|
+
`,
|
|
1168
|
+
args: {
|
|
1169
|
+
queryVector,
|
|
1170
|
+
indexName: embeddingIndexName,
|
|
1171
|
+
candidates: candidateCount,
|
|
1172
|
+
resultLimit
|
|
1173
|
+
}
|
|
1174
|
+
});
|
|
1175
|
+
} catch (error) {
|
|
1176
|
+
throw wrapIndexPathError(error, embeddingIndexName);
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
async function executeExactSearch(client, quotedTableName, queryVector, resultLimit) {
|
|
1180
|
+
return client.execute({
|
|
960
1181
|
sql: `
|
|
961
1182
|
SELECT
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
created_at,
|
|
969
|
-
vector_distance_cos(embedding, vector(?)) as distance
|
|
970
|
-
FROM ${quotedTableName}
|
|
971
|
-
WHERE embedding IS NOT NULL
|
|
972
|
-
ORDER BY distance
|
|
973
|
-
LIMIT ?
|
|
1183
|
+
${RESULT_COLUMNS},
|
|
1184
|
+
vector_distance_cos(a.embedding, vector(:queryVector)) as distance
|
|
1185
|
+
FROM ${quotedTableName} a
|
|
1186
|
+
WHERE a.embedding IS NOT NULL
|
|
1187
|
+
ORDER BY distance, a.id
|
|
1188
|
+
LIMIT :resultLimit
|
|
974
1189
|
`,
|
|
975
|
-
args:
|
|
1190
|
+
args: { queryVector, resultLimit }
|
|
976
1191
|
});
|
|
977
|
-
|
|
1192
|
+
}
|
|
1193
|
+
function wrapIndexPathError(error, embeddingIndexName) {
|
|
1194
|
+
if (!(error instanceof Error)) {
|
|
1195
|
+
return error;
|
|
1196
|
+
}
|
|
1197
|
+
if (MISSING_VECTOR_INDEX_PATTERNS.some((pattern) => pattern.test(error.message))) {
|
|
1198
|
+
return new Error(
|
|
1199
|
+
`Vector index "${embeddingIndexName}" could not be used for search. createTable() creates this index; a table created before it existed, or created by hand, will not have it. Create it with CREATE INDEX IF NOT EXISTS "${embeddingIndexName}" ON <table>(libsql_vector_idx(embedding)), or pass exact: true to search without the index.`,
|
|
1200
|
+
{ cause: error }
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
if (NO_VECTOR_SUPPORT_PATTERNS.some((pattern) => pattern.test(error.message))) {
|
|
1204
|
+
return new Error(
|
|
1205
|
+
`This libSQL deployment has no vector index support: vector_top_k() is unavailable. Pass exact: true to search on the full-scan path.`,
|
|
1206
|
+
{ cause: error }
|
|
1207
|
+
);
|
|
1208
|
+
}
|
|
1209
|
+
return error;
|
|
1210
|
+
}
|
|
1211
|
+
function toSearchResult(row) {
|
|
1212
|
+
return {
|
|
978
1213
|
id: row.id,
|
|
979
1214
|
slug: row.slug,
|
|
980
1215
|
title: row.title,
|
|
@@ -983,7 +1218,7 @@ async function search(options) {
|
|
|
983
1218
|
tags: JSON.parse(row.tags || "[]"),
|
|
984
1219
|
distance: row.distance,
|
|
985
1220
|
created_at: row.created_at
|
|
986
|
-
}
|
|
1221
|
+
};
|
|
987
1222
|
}
|
|
988
1223
|
async function getAllArticles(client, tableName = "articles") {
|
|
989
1224
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
@@ -1057,4 +1292,4 @@ async function getFolders(client, tableName = "articles") {
|
|
|
1057
1292
|
return results.rows.map((row) => row.folder);
|
|
1058
1293
|
}
|
|
1059
1294
|
|
|
1060
|
-
export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
|
|
1295
|
+
export { DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, IndexingError, MAX_SEARCH_CANDIDATES, MIN_SEARCH_CANDIDATES, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
|
package/docs/API.md
CHANGED
|
@@ -18,6 +18,10 @@
|
|
|
18
18
|
- `validateEmbeddingBatch`
|
|
19
19
|
- `padEmbedding`
|
|
20
20
|
- `prepareTextForEmbedding`
|
|
21
|
+
- `IndexingError`
|
|
22
|
+
- `DEFAULT_SEARCH_CANDIDATE_MULTIPLIER`
|
|
23
|
+
- `MIN_SEARCH_CANDIDATES`
|
|
24
|
+
- `MAX_SEARCH_CANDIDATES`
|
|
21
25
|
|
|
22
26
|
It also exports these types:
|
|
23
27
|
|
|
@@ -34,6 +38,11 @@ It also exports these types:
|
|
|
34
38
|
- `EmbeddingOptions`
|
|
35
39
|
- `IndexerOptions`
|
|
36
40
|
- `IndexedDocument`
|
|
41
|
+
- `IndexResult`
|
|
42
|
+
- `IndexFailure`
|
|
43
|
+
- `IndexFailurePolicy`
|
|
44
|
+
- `IndexFailureStage`
|
|
45
|
+
- `IndexingErrorPhase`
|
|
37
46
|
- `SearchOptions`
|
|
38
47
|
- `SearchResult`
|
|
39
48
|
|
|
@@ -86,6 +95,8 @@ interface IndexerOptions {
|
|
|
86
95
|
exclude?: string[];
|
|
87
96
|
tableName?: string;
|
|
88
97
|
onProgress?: (current: number, total: number, file: string) => void;
|
|
98
|
+
failurePolicy?: "abort" | "skip";
|
|
99
|
+
allowEmptyIndex?: boolean;
|
|
89
100
|
}
|
|
90
101
|
```
|
|
91
102
|
|
|
@@ -94,25 +105,80 @@ Defaults:
|
|
|
94
105
|
- `fileExtensions`: [".md", ".markdown"]
|
|
95
106
|
- `exclude`: ["node_modules", ".git", "dist", "build"]
|
|
96
107
|
- `tableName`: `"articles"`
|
|
108
|
+
- `failurePolicy`: `"abort"`
|
|
109
|
+
- `allowEmptyIndex`: `false`
|
|
97
110
|
|
|
98
111
|
Return shape:
|
|
99
112
|
|
|
100
113
|
```ts
|
|
101
|
-
{
|
|
102
|
-
success: number;
|
|
103
|
-
failed: number;
|
|
104
|
-
total: number;
|
|
114
|
+
interface IndexResult {
|
|
115
|
+
success: number; // documents written
|
|
116
|
+
failed: number; // files that could not be indexed
|
|
117
|
+
total: number; // files discovered on disk
|
|
118
|
+
replaced: boolean; // whether table contents were replaced by this call
|
|
119
|
+
partial: boolean; // replaced, but some files were skipped
|
|
120
|
+
failures: IndexFailure[];
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
interface IndexFailure {
|
|
124
|
+
file: string; // path relative to contentPath
|
|
125
|
+
stage: "read" | "parse" | "embed";
|
|
126
|
+
error: Error;
|
|
105
127
|
}
|
|
106
128
|
```
|
|
107
129
|
|
|
108
130
|
Behavior notes:
|
|
109
131
|
|
|
110
|
-
-
|
|
111
|
-
-
|
|
132
|
+
- every file is read, parsed, and embedded in memory before any database state changes
|
|
133
|
+
- the target table is then replaced in a single write transaction, so a failed rebuild leaves the previous index exactly as it was
|
|
134
|
+
- that costs peak memory proportional to the whole corpus, and against remote clients the replacement travels as a single un-chunked batch request; see [Costs of the two-phase rebuild](./INDEXING.md#costs-of-the-two-phase-rebuild) before rebuilding a very large corpus in place
|
|
135
|
+
- files are discovered and indexed in sorted path order
|
|
136
|
+
- frontmatter `title` must be a scalar; a structured title such as a YAML list fails the file at the `parse` stage
|
|
137
|
+
- two files that reduce to the same slug (`foo.md` and `foo.markdown`) collide: the first in sorted path order keeps the slug and the later file is reported as a `parse` failure
|
|
138
|
+
- `failurePolicy: "abort"` throws `IndexingError` on the first file that fails
|
|
139
|
+
- `failurePolicy: "skip"` drops the failing file, records it in `failures`, and rebuilds from the survivors, returning `partial: true`
|
|
140
|
+
- under `"skip"`, if every discovered file fails, the rebuild throws instead of replacing a valid index with an empty one
|
|
141
|
+
- an empty source directory throws unless `allowEmptyIndex: true`, which intentionally empties the index
|
|
142
|
+
- `onProgress` is called once per file during the build phase
|
|
112
143
|
- frontmatter `title`, `description`, and `tags` are folded into the embedding text
|
|
113
144
|
- embeddings default to `intent: "document"` unless `embeddingOptions.intent` is set explicitly
|
|
114
145
|
- if a file has no frontmatter title, the filename becomes the title
|
|
115
146
|
|
|
147
|
+
### `IndexingError`
|
|
148
|
+
|
|
149
|
+
Thrown when a rebuild cannot complete. The previously indexed rows are always left unchanged.
|
|
150
|
+
|
|
151
|
+
```ts
|
|
152
|
+
class IndexingError extends Error {
|
|
153
|
+
readonly phase: "build" | "replace";
|
|
154
|
+
readonly failures: IndexFailure[];
|
|
155
|
+
}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
- `phase: "build"` means the failure happened before any database work: a file failed, every file failed, the source directory was empty, or it could not be scanned
|
|
159
|
+
- `phase: "replace"` means the replacement transaction failed and was rolled back
|
|
160
|
+
- `cause` carries the underlying error
|
|
161
|
+
- on a `phase: "replace"` error, `failures` lists files skipped during the build phase. They are not the cause of the rollback, which is carried by `cause`
|
|
162
|
+
|
|
163
|
+
```ts
|
|
164
|
+
import { indexContent, IndexingError } from "libsql-search";
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
await indexContent({ client, contentPath: "./content" });
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (error instanceof IndexingError) {
|
|
170
|
+
console.error(error.phase, error.failures);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Breaking changes in this behavior:
|
|
178
|
+
|
|
179
|
+
- partial failures previously counted into `failed` and still replaced the table; they now throw. Pass `failurePolicy: "skip"` for the previous lenient behavior.
|
|
180
|
+
- an empty source directory previously returned zeros and left stale rows in place; it now throws. Pass `allowEmptyIndex: true` to intentionally empty the index.
|
|
181
|
+
|
|
116
182
|
## `search(options)`
|
|
117
183
|
|
|
118
184
|
Generates a query embedding and performs vector similarity search.
|
|
@@ -124,6 +190,8 @@ interface SearchOptions {
|
|
|
124
190
|
limit?: number;
|
|
125
191
|
tableName?: string;
|
|
126
192
|
embeddingOptions?: EmbeddingOptions;
|
|
193
|
+
candidates?: number;
|
|
194
|
+
exact?: boolean;
|
|
127
195
|
}
|
|
128
196
|
```
|
|
129
197
|
|
|
@@ -131,9 +199,65 @@ Defaults:
|
|
|
131
199
|
|
|
132
200
|
- `limit`: `10`
|
|
133
201
|
- `tableName`: `"articles"`
|
|
202
|
+
- `candidates`: `Math.max(limit * 4, 32)`
|
|
203
|
+
- `exact`: `false`
|
|
134
204
|
|
|
135
205
|
`limit` must be an integer from `1` through `100`; invalid values are rejected before query embedding generation.
|
|
136
206
|
|
|
207
|
+
### Search Is Approximate By Default
|
|
208
|
+
|
|
209
|
+
The default path queries the `<tableName>_embedding_idx` vector index through libSQL's `vector_top_k()`. **That index is an approximate-nearest-neighbor structure. It can miss a true nearest neighbor.** There is no configuration that makes the index path exact; only `exact: true` guarantees exactness.
|
|
210
|
+
|
|
211
|
+
To limit the accuracy loss, the default path over-fetches: it pulls `candidates` rows from the index, recomputes the true `vector_distance_cos` for each, and orders exactly by `(distance, id)` before trimming to `limit`. So:
|
|
212
|
+
|
|
213
|
+
- **Recall is approximate.** A row the index does not return as a candidate cannot appear in the results, no matter its true distance. Raising `candidates` raises recall.
|
|
214
|
+
- **Ranking within the candidate set is exact.** Distances on returned rows are always the true cosine distances, not index approximations.
|
|
215
|
+
- **Ordering is fully deterministic.** The `id` tiebreaker means two rows at an identical distance always come back in the same order, even though the index's own candidate order is not stable across runs.
|
|
216
|
+
- **Rows with a `NULL` embedding are never returned.** The vector index excludes them on its own.
|
|
217
|
+
|
|
218
|
+
**Both paths order by `(distance, id)`.** The determinism guarantee covers `exact: true` as well as the default, so the two paths return identical orderings for identical inputs and can be compared directly. This is a change in tie ordering for the exact path, which previously sorted by distance alone and left tied rows in whatever order the scan produced.
|
|
219
|
+
|
|
220
|
+
### `candidates`
|
|
221
|
+
|
|
222
|
+
Controls how many rows the index returns for the exact re-rank. It must be an integer from `limit` through `MAX_SEARCH_CANDIDATES`; a value below `limit` is rejected rather than clamped, because it would silently truncate the result set. Invalid values throw before query embedding generation and before any database call.
|
|
223
|
+
|
|
224
|
+
```ts
|
|
225
|
+
// Trade query cost for recall on a large corpus
|
|
226
|
+
const results = await search({ client, query, limit: 10, candidates: 200 });
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
`candidates` has no effect when `exact` is `true` — that path scans every row — but it is **still validated**. `search({ exact: true, limit: 10, candidates: 5 })` throws, exactly as it would on the index path. Validity does not depend on which path a call happens to take.
|
|
230
|
+
|
|
231
|
+
Related exported constants:
|
|
232
|
+
|
|
233
|
+
- `DEFAULT_SEARCH_CANDIDATE_MULTIPLIER` (`4`): multiplier applied to `limit`
|
|
234
|
+
- `MIN_SEARCH_CANDIDATES` (`32`): floor for the derived default
|
|
235
|
+
- `MAX_SEARCH_CANDIDATES` (`1000`): ceiling for any explicit value. The derived default cannot reach it today, since `limit` tops out at `100`; the cap constrains explicit values.
|
|
236
|
+
|
|
237
|
+
### `exact`
|
|
238
|
+
|
|
239
|
+
Set `exact: true` to bypass the index and score every row in the table.
|
|
240
|
+
|
|
241
|
+
```ts
|
|
242
|
+
const results = await search({ client, query, exact: true });
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
This is the guaranteed-exact path: it computes `vector_distance_cos` for every row with a non-`NULL` embedding, sorts by `(distance, id)`, and trims to `limit`. Cost grows linearly with table size, so it is intended for small corpora, correctness checks against the index path, and tables that have no vector index.
|
|
246
|
+
|
|
247
|
+
### Missing Vector Index
|
|
248
|
+
|
|
249
|
+
If the target table has no `<tableName>_embedding_idx`, the default path throws an error naming the missing index and pointing at `createTable()` and `exact: true`. libSQL's own message for this case ("failed to parse vector index parameters") says nothing about a missing index, so it is preserved as the thrown error's `cause` rather than surfaced directly.
|
|
250
|
+
|
|
251
|
+
A deployment with no vector support at all is a separate case with its own message. `vector_top_k()` does not exist there, so no `CREATE INDEX` can help and the error points only at `exact: true`.
|
|
252
|
+
|
|
253
|
+
**Those two messages are the only ones rewritten.** Every other failure from the index path reaches the caller with its original message intact. In particular, a width mismatch between the query embedding and the `embedding` column surfaces as libSQL's own `vector index(search): dimensions are different: 384 != 4`, which names both widths and is the useful diagnostic for the dimension drift described in the [Migration and reindexing guide](./MIGRATIONS.md).
|
|
254
|
+
|
|
255
|
+
Search never falls back to the exact scan on its own. A silent fallback would turn a one-line schema fix into an invisible, permanent full-table scan.
|
|
256
|
+
|
|
257
|
+
### Requirements
|
|
258
|
+
|
|
259
|
+
`vector_top_k()` and `libsql_vector_idx()` require a libSQL build with native vector support. The peer dependency is `@libsql/client ^0.15.0`; this behavior is verified against `@libsql/client` `0.15.15` with a local `:memory:` database. Remote Turso/libSQL servers must also support vector indexes — that is a property of the server, not the client, and no minimum server version is claimed here beyond that requirement. A deployment without it fails the default path with an error saying so and naming `exact: true` as the remedy. Use `exact: true` against any deployment where vector index support is unavailable or unverified.
|
|
260
|
+
|
|
137
261
|
Result shape:
|
|
138
262
|
|
|
139
263
|
```ts
|