libsql-search 0.9.1 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.cjs +95 -40
- package/dist/index.d.ts +107 -8
- package/dist/index.esm.js +95 -40
- package/dist/turso.cjs +96 -0
- package/dist/turso.d.ts +156 -0
- package/dist/turso.esm.js +94 -0
- package/docs/API.md +28 -3
- package/docs/README.md +1 -0
- package/docs/TURSO.md +230 -0
- package/package.json +16 -3
package/dist/index.esm.js
CHANGED
|
@@ -831,6 +831,48 @@ function normalizeVectorDimensions(dimensions) {
|
|
|
831
831
|
return dimensions;
|
|
832
832
|
}
|
|
833
833
|
|
|
834
|
+
function isDatabaseAdapter(client) {
|
|
835
|
+
return typeof client === "object" && client !== null && client.libsqlSearchAdapter === true;
|
|
836
|
+
}
|
|
837
|
+
function resolveDatabase(client) {
|
|
838
|
+
if (isDatabaseAdapter(client)) {
|
|
839
|
+
assertCompleteAdapter(client);
|
|
840
|
+
return client;
|
|
841
|
+
}
|
|
842
|
+
return createLibsqlAdapter(client);
|
|
843
|
+
}
|
|
844
|
+
const ADAPTER_METHODS = ["executeDdl", "executeQuery", "executeAtomicWrite"];
|
|
845
|
+
function assertCompleteAdapter(adapter) {
|
|
846
|
+
const missing = ADAPTER_METHODS.filter(
|
|
847
|
+
(method) => typeof adapter[method] !== "function"
|
|
848
|
+
);
|
|
849
|
+
if (missing.length > 0) {
|
|
850
|
+
throw new TypeError(
|
|
851
|
+
`This client is marked as a libsql-search database adapter but is missing ${missing.join(", ")}. The usual cause is two different versions of libsql-search resolved in one dependency tree, so the adapter was built by a different copy of the package than the one calling it. Deduplicate libsql-search, or build the adapter from the same copy you call.`
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
function createLibsqlAdapter(client) {
|
|
856
|
+
return {
|
|
857
|
+
libsqlSearchAdapter: true,
|
|
858
|
+
backend: "libsql",
|
|
859
|
+
supportsVectorIndex: true,
|
|
860
|
+
async executeDdl(sql) {
|
|
861
|
+
await client.execute(sql);
|
|
862
|
+
},
|
|
863
|
+
async executeQuery(sql, args) {
|
|
864
|
+
const result = args === void 0 ? await client.execute(sql) : await client.execute({ sql, args });
|
|
865
|
+
return result.rows;
|
|
866
|
+
},
|
|
867
|
+
async executeAtomicWrite(statements) {
|
|
868
|
+
const batch = statements.map(
|
|
869
|
+
(statement) => statement.args === void 0 ? statement.sql : statement
|
|
870
|
+
);
|
|
871
|
+
await client.batch(batch, "write");
|
|
872
|
+
}
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
|
|
834
876
|
class IndexingError extends Error {
|
|
835
877
|
phase;
|
|
836
878
|
failures;
|
|
@@ -853,6 +895,7 @@ async function indexContent(options) {
|
|
|
853
895
|
failurePolicy = "abort",
|
|
854
896
|
allowEmptyIndex = false
|
|
855
897
|
} = options;
|
|
898
|
+
const database = resolveDatabase(client);
|
|
856
899
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
857
900
|
let files;
|
|
858
901
|
try {
|
|
@@ -926,7 +969,7 @@ async function indexContent(options) {
|
|
|
926
969
|
);
|
|
927
970
|
}
|
|
928
971
|
const success = documents.length;
|
|
929
|
-
await replaceIndex(
|
|
972
|
+
await replaceIndex(database, quotedTableName, documents, failures);
|
|
930
973
|
return {
|
|
931
974
|
success,
|
|
932
975
|
failed: failures.length,
|
|
@@ -972,11 +1015,13 @@ async function buildDocument(file, embeddingOptions) {
|
|
|
972
1015
|
return { ok: false, stage: "parse", error: toError(error) };
|
|
973
1016
|
}
|
|
974
1017
|
let embedding;
|
|
1018
|
+
let serializedEmbedding;
|
|
975
1019
|
try {
|
|
976
1020
|
embedding = await generateEmbedding(parsed.embeddingText, {
|
|
977
1021
|
...embeddingOptions,
|
|
978
1022
|
intent: embeddingOptions.intent ?? "document"
|
|
979
1023
|
});
|
|
1024
|
+
serializedEmbedding = JSON.stringify(embedding);
|
|
980
1025
|
} catch (error) {
|
|
981
1026
|
return { ok: false, stage: "embed", error: toError(error) };
|
|
982
1027
|
}
|
|
@@ -990,6 +1035,7 @@ async function buildDocument(file, embeddingOptions) {
|
|
|
990
1035
|
tags: parsed.tags,
|
|
991
1036
|
serializedTags: parsed.serializedTags,
|
|
992
1037
|
embedding,
|
|
1038
|
+
serializedEmbedding,
|
|
993
1039
|
metadata: parsed.metadata
|
|
994
1040
|
}
|
|
995
1041
|
};
|
|
@@ -1039,8 +1085,8 @@ function fallbackTitle(file) {
|
|
|
1039
1085
|
function describeType(value) {
|
|
1040
1086
|
return Array.isArray(value) ? "array" : typeof value;
|
|
1041
1087
|
}
|
|
1042
|
-
async function replaceIndex(
|
|
1043
|
-
const statements = [`DELETE FROM ${quotedTableName}`];
|
|
1088
|
+
async function replaceIndex(database, quotedTableName, documents, failures) {
|
|
1089
|
+
const statements = [{ sql: `DELETE FROM ${quotedTableName}` }];
|
|
1044
1090
|
try {
|
|
1045
1091
|
for (let i = 0; i < documents.length; i++) {
|
|
1046
1092
|
const document = documents[i];
|
|
@@ -1053,7 +1099,7 @@ async function replaceIndex(client, quotedTableName, documents, failures) {
|
|
|
1053
1099
|
documents.length = 0;
|
|
1054
1100
|
}
|
|
1055
1101
|
try {
|
|
1056
|
-
await
|
|
1102
|
+
await database.executeAtomicWrite(statements);
|
|
1057
1103
|
} catch (error) {
|
|
1058
1104
|
throw new IndexingError(
|
|
1059
1105
|
`Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
|
|
@@ -1074,7 +1120,7 @@ function createInsertStatement(document, quotedTableName) {
|
|
|
1074
1120
|
document.content,
|
|
1075
1121
|
document.folder,
|
|
1076
1122
|
document.serializedTags,
|
|
1077
|
-
|
|
1123
|
+
document.serializedEmbedding
|
|
1078
1124
|
]
|
|
1079
1125
|
};
|
|
1080
1126
|
}
|
|
@@ -1082,12 +1128,13 @@ function toError(error) {
|
|
|
1082
1128
|
return error instanceof Error ? error : new Error(String(error));
|
|
1083
1129
|
}
|
|
1084
1130
|
async function createTable(client, tableName = "articles", dimensions = 384) {
|
|
1131
|
+
const database = resolveDatabase(client);
|
|
1085
1132
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1086
1133
|
const vectorDimensions = normalizeVectorDimensions(dimensions);
|
|
1087
1134
|
const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");
|
|
1088
1135
|
const quotedFolderIndexName = quoteSqlIdentifier(`${tableName}_folder_idx`, "folder index name");
|
|
1089
1136
|
const quotedSlugIndexName = quoteSqlIdentifier(`${tableName}_slug_idx`, "slug index name");
|
|
1090
|
-
await
|
|
1137
|
+
await database.executeDdl(`
|
|
1091
1138
|
CREATE TABLE IF NOT EXISTS ${quotedTableName} (
|
|
1092
1139
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1093
1140
|
slug TEXT UNIQUE NOT NULL,
|
|
@@ -1100,15 +1147,17 @@ async function createTable(client, tableName = "articles", dimensions = 384) {
|
|
|
1100
1147
|
updated_at TEXT NOT NULL
|
|
1101
1148
|
)
|
|
1102
1149
|
`);
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1150
|
+
if (database.supportsVectorIndex) {
|
|
1151
|
+
await database.executeDdl(`
|
|
1152
|
+
CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
|
|
1153
|
+
ON ${quotedTableName}(libsql_vector_idx(embedding))
|
|
1154
|
+
`);
|
|
1155
|
+
}
|
|
1156
|
+
await database.executeDdl(`
|
|
1108
1157
|
CREATE INDEX IF NOT EXISTS ${quotedFolderIndexName}
|
|
1109
1158
|
ON ${quotedTableName}(folder)
|
|
1110
1159
|
`);
|
|
1111
|
-
await
|
|
1160
|
+
await database.executeDdl(`
|
|
1112
1161
|
CREATE INDEX IF NOT EXISTS ${quotedSlugIndexName}
|
|
1113
1162
|
ON ${quotedTableName}(slug)
|
|
1114
1163
|
`);
|
|
@@ -1131,6 +1180,7 @@ async function search(options) {
|
|
|
1131
1180
|
candidates,
|
|
1132
1181
|
exact = false
|
|
1133
1182
|
} = options;
|
|
1183
|
+
const database = resolveDatabase(client);
|
|
1134
1184
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1135
1185
|
const resultLimit = normalizeSearchLimit(limit);
|
|
1136
1186
|
const embeddingIndexName = validateSqlIdentifier(
|
|
@@ -1143,20 +1193,21 @@ async function search(options) {
|
|
|
1143
1193
|
intent: embeddingOptions.intent ?? "query"
|
|
1144
1194
|
});
|
|
1145
1195
|
const queryVector = JSON.stringify(queryEmbedding);
|
|
1146
|
-
const
|
|
1147
|
-
|
|
1196
|
+
const useExactSearch = exact || !database.supportsVectorIndex;
|
|
1197
|
+
const rows = useExactSearch ? await executeExactSearch(database, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
|
|
1198
|
+
database,
|
|
1148
1199
|
quotedTableName,
|
|
1149
1200
|
embeddingIndexName,
|
|
1150
1201
|
queryVector,
|
|
1151
1202
|
candidateCount,
|
|
1152
1203
|
resultLimit
|
|
1153
1204
|
);
|
|
1154
|
-
return
|
|
1205
|
+
return rows.map(toSearchResult);
|
|
1155
1206
|
}
|
|
1156
|
-
async function executeIndexSearch(
|
|
1207
|
+
async function executeIndexSearch(database, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
|
|
1157
1208
|
try {
|
|
1158
|
-
return await
|
|
1159
|
-
|
|
1209
|
+
return await database.executeQuery(
|
|
1210
|
+
`
|
|
1160
1211
|
SELECT
|
|
1161
1212
|
${RESULT_COLUMNS},
|
|
1162
1213
|
vector_distance_cos(a.embedding, vector(:queryVector)) as distance
|
|
@@ -1165,20 +1216,20 @@ async function executeIndexSearch(client, quotedTableName, embeddingIndexName, q
|
|
|
1165
1216
|
ORDER BY distance, a.id
|
|
1166
1217
|
LIMIT :resultLimit
|
|
1167
1218
|
`,
|
|
1168
|
-
|
|
1219
|
+
{
|
|
1169
1220
|
queryVector,
|
|
1170
1221
|
indexName: embeddingIndexName,
|
|
1171
1222
|
candidates: candidateCount,
|
|
1172
1223
|
resultLimit
|
|
1173
1224
|
}
|
|
1174
|
-
|
|
1225
|
+
);
|
|
1175
1226
|
} catch (error) {
|
|
1176
1227
|
throw wrapIndexPathError(error, embeddingIndexName);
|
|
1177
1228
|
}
|
|
1178
1229
|
}
|
|
1179
|
-
async function executeExactSearch(
|
|
1180
|
-
return
|
|
1181
|
-
|
|
1230
|
+
async function executeExactSearch(database, quotedTableName, queryVector, resultLimit) {
|
|
1231
|
+
return database.executeQuery(
|
|
1232
|
+
`
|
|
1182
1233
|
SELECT
|
|
1183
1234
|
${RESULT_COLUMNS},
|
|
1184
1235
|
vector_distance_cos(a.embedding, vector(:queryVector)) as distance
|
|
@@ -1187,8 +1238,8 @@ async function executeExactSearch(client, quotedTableName, queryVector, resultLi
|
|
|
1187
1238
|
ORDER BY distance, a.id
|
|
1188
1239
|
LIMIT :resultLimit
|
|
1189
1240
|
`,
|
|
1190
|
-
|
|
1191
|
-
|
|
1241
|
+
{ queryVector, resultLimit }
|
|
1242
|
+
);
|
|
1192
1243
|
}
|
|
1193
1244
|
function wrapIndexPathError(error, embeddingIndexName) {
|
|
1194
1245
|
if (!(error instanceof Error)) {
|
|
@@ -1221,13 +1272,14 @@ function toSearchResult(row) {
|
|
|
1221
1272
|
};
|
|
1222
1273
|
}
|
|
1223
1274
|
async function getAllArticles(client, tableName = "articles") {
|
|
1275
|
+
const database = resolveDatabase(client);
|
|
1224
1276
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1225
|
-
const
|
|
1277
|
+
const rows = await database.executeQuery(`
|
|
1226
1278
|
SELECT id, slug, title, folder, tags, created_at, updated_at
|
|
1227
1279
|
FROM ${quotedTableName}
|
|
1228
1280
|
ORDER BY title
|
|
1229
1281
|
`);
|
|
1230
|
-
return
|
|
1282
|
+
return rows.map((row) => ({
|
|
1231
1283
|
id: row.id,
|
|
1232
1284
|
slug: row.slug,
|
|
1233
1285
|
title: row.title,
|
|
@@ -1238,20 +1290,21 @@ async function getAllArticles(client, tableName = "articles") {
|
|
|
1238
1290
|
}));
|
|
1239
1291
|
}
|
|
1240
1292
|
async function getArticleBySlug(client, slug, tableName = "articles") {
|
|
1293
|
+
const database = resolveDatabase(client);
|
|
1241
1294
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1242
|
-
const
|
|
1243
|
-
|
|
1295
|
+
const rows = await database.executeQuery(
|
|
1296
|
+
`
|
|
1244
1297
|
SELECT id, slug, title, content, folder, tags, created_at, updated_at
|
|
1245
1298
|
FROM ${quotedTableName}
|
|
1246
1299
|
WHERE slug = ?
|
|
1247
1300
|
LIMIT 1
|
|
1248
1301
|
`,
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
if (
|
|
1302
|
+
[slug]
|
|
1303
|
+
);
|
|
1304
|
+
if (rows.length === 0) {
|
|
1252
1305
|
return null;
|
|
1253
1306
|
}
|
|
1254
|
-
const row =
|
|
1307
|
+
const row = rows[0];
|
|
1255
1308
|
return {
|
|
1256
1309
|
id: row.id,
|
|
1257
1310
|
slug: row.slug,
|
|
@@ -1264,17 +1317,18 @@ async function getArticleBySlug(client, slug, tableName = "articles") {
|
|
|
1264
1317
|
};
|
|
1265
1318
|
}
|
|
1266
1319
|
async function getArticlesByFolder(client, folder, tableName = "articles") {
|
|
1320
|
+
const database = resolveDatabase(client);
|
|
1267
1321
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1268
|
-
const
|
|
1269
|
-
|
|
1322
|
+
const rows = await database.executeQuery(
|
|
1323
|
+
`
|
|
1270
1324
|
SELECT id, slug, title, folder, tags
|
|
1271
1325
|
FROM ${quotedTableName}
|
|
1272
1326
|
WHERE folder = ?
|
|
1273
1327
|
ORDER BY title
|
|
1274
1328
|
`,
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
return
|
|
1329
|
+
[folder]
|
|
1330
|
+
);
|
|
1331
|
+
return rows.map((row) => ({
|
|
1278
1332
|
id: row.id,
|
|
1279
1333
|
slug: row.slug,
|
|
1280
1334
|
title: row.title,
|
|
@@ -1283,13 +1337,14 @@ async function getArticlesByFolder(client, folder, tableName = "articles") {
|
|
|
1283
1337
|
}));
|
|
1284
1338
|
}
|
|
1285
1339
|
async function getFolders(client, tableName = "articles") {
|
|
1340
|
+
const database = resolveDatabase(client);
|
|
1286
1341
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
1287
|
-
const
|
|
1342
|
+
const rows = await database.executeQuery(`
|
|
1288
1343
|
SELECT DISTINCT folder
|
|
1289
1344
|
FROM ${quotedTableName}
|
|
1290
1345
|
ORDER BY folder
|
|
1291
1346
|
`);
|
|
1292
|
-
return
|
|
1347
|
+
return rows.map((row) => row.folder);
|
|
1293
1348
|
}
|
|
1294
1349
|
|
|
1295
1350
|
export { DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, IndexingError, MAX_SEARCH_CANDIDATES, MIN_SEARCH_CANDIDATES, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
|
package/dist/turso.cjs
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
function tursoAdapter(database) {
|
|
4
|
+
assertTursoDatabase(database);
|
|
5
|
+
return {
|
|
6
|
+
libsqlSearchAdapter: true,
|
|
7
|
+
backend: "turso",
|
|
8
|
+
/**
|
|
9
|
+
* Turso Database ships the vector functions but no ANN index. This is a
|
|
10
|
+
* design direction rather than a gap: the project shipped SIMD-accelerated
|
|
11
|
+
* exact search instead, and the DiskANN port remains an open backlog item.
|
|
12
|
+
*/
|
|
13
|
+
supportsVectorIndex: false,
|
|
14
|
+
async executeDdl(sql) {
|
|
15
|
+
await database.exec(sql);
|
|
16
|
+
},
|
|
17
|
+
async executeQuery(sql, args) {
|
|
18
|
+
const statement = database.prepare(sql);
|
|
19
|
+
try {
|
|
20
|
+
const rows = await (args === void 0 ? statement.all() : statement.all(args));
|
|
21
|
+
return rows;
|
|
22
|
+
} finally {
|
|
23
|
+
closeStatement(statement);
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
/**
|
|
27
|
+
* Replace the table contents inside one explicit transaction.
|
|
28
|
+
*
|
|
29
|
+
* `batch()` is deliberately not used. It is not transactional on this
|
|
30
|
+
* backend: a batch that fails part way through leaves the statements before
|
|
31
|
+
* the failure applied, which would silently turn a failed rebuild into a
|
|
32
|
+
* half-destroyed index. `BEGIN IMMEDIATE` takes the write lock up front and
|
|
33
|
+
* `ROLLBACK` restores the previous rows exactly.
|
|
34
|
+
*
|
|
35
|
+
* Statements are prepared once per distinct SQL string and rebound per row.
|
|
36
|
+
* An index rebuild is one `DELETE` followed by N identical `INSERT`s, so
|
|
37
|
+
* this turns N+1 prepares into exactly 2 regardless of corpus size.
|
|
38
|
+
*/
|
|
39
|
+
async executeAtomicWrite(statements) {
|
|
40
|
+
const preparedBySql = /* @__PURE__ */ new Map();
|
|
41
|
+
const prepareOnce = (sql) => {
|
|
42
|
+
let prepared = preparedBySql.get(sql);
|
|
43
|
+
if (prepared === void 0) {
|
|
44
|
+
prepared = database.prepare(sql);
|
|
45
|
+
preparedBySql.set(sql, prepared);
|
|
46
|
+
}
|
|
47
|
+
return prepared;
|
|
48
|
+
};
|
|
49
|
+
await database.exec("BEGIN IMMEDIATE");
|
|
50
|
+
try {
|
|
51
|
+
for (const statement of statements) {
|
|
52
|
+
const prepared = prepareOnce(statement.sql);
|
|
53
|
+
await (statement.args === void 0 ? prepared.run() : prepared.run(statement.args));
|
|
54
|
+
}
|
|
55
|
+
await database.exec("COMMIT");
|
|
56
|
+
} catch (error) {
|
|
57
|
+
try {
|
|
58
|
+
await database.exec("ROLLBACK");
|
|
59
|
+
} catch (rollbackError) {
|
|
60
|
+
warnOnFailedRollback(rollbackError);
|
|
61
|
+
}
|
|
62
|
+
throw error;
|
|
63
|
+
} finally {
|
|
64
|
+
for (const prepared of preparedBySql.values()) {
|
|
65
|
+
closeStatement(prepared);
|
|
66
|
+
}
|
|
67
|
+
preparedBySql.clear();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function closeStatement(statement) {
|
|
73
|
+
try {
|
|
74
|
+
statement.close?.();
|
|
75
|
+
} catch {
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const NO_ACTIVE_TRANSACTION_PATTERN = /no transaction is active/i;
|
|
79
|
+
function warnOnFailedRollback(error) {
|
|
80
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
81
|
+
if (NO_ACTIVE_TRANSACTION_PATTERN.test(message)) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
console.warn(
|
|
85
|
+
`[libsql-search] ROLLBACK failed after a failed index replacement: ${message}. This database handle may still be inside an open transaction, in which case a later write on it fails with "cannot start a transaction within a transaction". Reconnect the handle before reusing it.`
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
function assertTursoDatabase(database) {
|
|
89
|
+
if (typeof database !== "object" || database === null || typeof database.exec !== "function" || typeof database.prepare !== "function") {
|
|
90
|
+
throw new TypeError(
|
|
91
|
+
"tursoAdapter() expects a connected @tursodatabase/database handle with exec() and prepare() methods. connect() returns a Promise, so remember to await it."
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
exports.tursoAdapter = tursoAdapter;
|
package/dist/turso.d.ts
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Database backend boundary.
|
|
3
|
+
*
|
|
4
|
+
* Every SQL call this library makes goes through {@link DatabaseAdapter}. The
|
|
5
|
+
* `@libsql/client` adapter is built here, because that client is the declared
|
|
6
|
+
* peer dependency and the only backend the main entry point knows about. Other
|
|
7
|
+
* backends supply their own adapter from their own entry point, so the main
|
|
8
|
+
* bundle never imports, resolves, or references their package.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The narrow database contract the indexer, search, and retrieval helpers use.
|
|
13
|
+
*
|
|
14
|
+
* This interface is deliberately self-contained: it names no other type from
|
|
15
|
+
* this package and no type from any backend package. That keeps the declaration
|
|
16
|
+
* emitted into every entry point's `.d.ts` structurally identical, so an adapter
|
|
17
|
+
* built by one entry point is assignable to the parameter types of another
|
|
18
|
+
* without the two entry points sharing a nominal symbol.
|
|
19
|
+
*
|
|
20
|
+
* Do not brand this with a `unique symbol`. Each bundled `.d.ts` declares its
|
|
21
|
+
* own copy of the interface, and a `unique symbol` would make those copies
|
|
22
|
+
* nominally distinct — an adapter from the `libsql-search/turso` entry point
|
|
23
|
+
* would stop being assignable to `SearchOptions['client']`.
|
|
24
|
+
*/
|
|
25
|
+
interface DatabaseAdapter {
|
|
26
|
+
/**
|
|
27
|
+
* Marker set by this package's adapter factories.
|
|
28
|
+
*
|
|
29
|
+
* Public functions accept either a raw `@libsql/client` `Client` or an
|
|
30
|
+
* adapter, and this property is how they tell the two apart. It is a plain
|
|
31
|
+
* property rather than structural method sniffing on purpose: guessing a
|
|
32
|
+
* backend from its method names would risk picking the wrong atomicity
|
|
33
|
+
* primitive, and the two backends disagree about which primitive is
|
|
34
|
+
* transactional.
|
|
35
|
+
*/
|
|
36
|
+
readonly libsqlSearchAdapter: true;
|
|
37
|
+
/** Which backend this adapter drives. Informational; behavior keys off the capability flags. */
|
|
38
|
+
readonly backend: string;
|
|
39
|
+
/**
|
|
40
|
+
* Whether the backend can build a `libsql_vector_idx()` index and query it
|
|
41
|
+
* through `vector_top_k()`.
|
|
42
|
+
*
|
|
43
|
+
* When false, `createTable()` skips the vector index and `search()` runs the
|
|
44
|
+
* exact full-scan path automatically. Both are required, not optimizations:
|
|
45
|
+
* on a backend without the index, `CREATE INDEX ... libsql_vector_idx(...)`
|
|
46
|
+
* is a parse error and `vector_top_k` is not a table.
|
|
47
|
+
*/
|
|
48
|
+
readonly supportsVectorIndex: boolean;
|
|
49
|
+
/** Run a schema statement that returns no rows. */
|
|
50
|
+
executeDdl(sql: string): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Run one statement and return its rows as plain column-keyed objects.
|
|
53
|
+
*
|
|
54
|
+
* The bind-value union is spelled out inline rather than named, so that this
|
|
55
|
+
* interface stays free of any other symbol. It is narrower than `unknown` on
|
|
56
|
+
* purpose: it is the only compile-time check that a caller is binding
|
|
57
|
+
* something a SQLite driver can actually store.
|
|
58
|
+
*/
|
|
59
|
+
executeQuery(sql: string, args?: Readonly<Record<string, string | number | bigint | Uint8Array | null>> | ReadonlyArray<string | number | bigint | Uint8Array | null>): Promise<Array<Record<string, unknown>>>;
|
|
60
|
+
/**
|
|
61
|
+
* Apply every statement atomically: either all of them commit, or none do.
|
|
62
|
+
*
|
|
63
|
+
* The whole "a failed rebuild leaves the previous index intact" guarantee
|
|
64
|
+
* rests on this method. A backend whose batch primitive is not transactional
|
|
65
|
+
* must implement this with an explicit transaction instead.
|
|
66
|
+
*/
|
|
67
|
+
executeAtomicWrite(statements: ReadonlyArray<{
|
|
68
|
+
sql: string;
|
|
69
|
+
args?: ReadonlyArray<string | number | bigint | Uint8Array | null>;
|
|
70
|
+
}>): Promise<void>;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Turso Database adapter — `libsql-search/turso`.
|
|
75
|
+
*
|
|
76
|
+
* **Experimental, and exact-search only.** See `docs/TURSO.md`.
|
|
77
|
+
*
|
|
78
|
+
* This entry point exists so that the main entry point never mentions
|
|
79
|
+
* `@tursodatabase/database`. Nothing here imports it either: the database
|
|
80
|
+
* handle is accepted through the structural {@link TursoDatabase} type, so
|
|
81
|
+
* neither `tsc`, nor Deno, nor a bundler is ever asked to resolve the
|
|
82
|
+
* native package on behalf of a user who does not use it.
|
|
83
|
+
*
|
|
84
|
+
* ```ts
|
|
85
|
+
* import { connect } from '@tursodatabase/database';
|
|
86
|
+
* import { tursoAdapter } from 'libsql-search/turso';
|
|
87
|
+
* import { createTable, indexContent, search } from 'libsql-search';
|
|
88
|
+
*
|
|
89
|
+
* const client = tursoAdapter(await connect('./local.db'));
|
|
90
|
+
*
|
|
91
|
+
* await createTable(client);
|
|
92
|
+
* await indexContent({ client, contentPath: './content' });
|
|
93
|
+
* const results = await search({ client, query: 'vector search' });
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* @module libsql-search/turso
|
|
97
|
+
*/
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The prepared-statement surface this adapter uses, as
|
|
101
|
+
* `@tursodatabase/database` exposes it.
|
|
102
|
+
*
|
|
103
|
+
* Every method returns a Promise on this backend, but they are typed as
|
|
104
|
+
* possibly-synchronous and always awaited, so a synchronous driver with the
|
|
105
|
+
* same shape works too.
|
|
106
|
+
*/
|
|
107
|
+
interface TursoStatement {
|
|
108
|
+
run(args?: unknown): unknown;
|
|
109
|
+
all(args?: unknown): unknown;
|
|
110
|
+
/**
|
|
111
|
+
* Release the native statement. Optional: a driver that manages statement
|
|
112
|
+
* lifetime itself may omit it.
|
|
113
|
+
*
|
|
114
|
+
* Not closing leaks roughly 10 KB of native memory per prepare on
|
|
115
|
+
* `@tursodatabase/database`, which the garbage collector does not reclaim
|
|
116
|
+
* because it is not JavaScript heap. A server calling `search()` per request
|
|
117
|
+
* grows without bound until the process is killed.
|
|
118
|
+
*/
|
|
119
|
+
close?(): unknown;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* The database-handle surface this adapter uses, matching the object returned
|
|
123
|
+
* by `connect()` from `@tursodatabase/database`.
|
|
124
|
+
*
|
|
125
|
+
* Declared structurally on purpose. Importing the real type would make
|
|
126
|
+
* `@tursodatabase/database` a hard resolution target for anyone who type-checks
|
|
127
|
+
* this package, including the many users who only ever touch `@libsql/client`.
|
|
128
|
+
*/
|
|
129
|
+
interface TursoDatabase {
|
|
130
|
+
exec(sql: string): unknown;
|
|
131
|
+
prepare(sql: string): TursoStatement;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Wrap a `@tursodatabase/database` handle so this library's functions can use
|
|
135
|
+
* it.
|
|
136
|
+
*
|
|
137
|
+
* The returned value is passed as `client` to `createTable()`,
|
|
138
|
+
* `indexContent()`, `search()`, and the retrieval helpers.
|
|
139
|
+
*
|
|
140
|
+
* Two behaviors differ from `@libsql/client`, and both are forced by the
|
|
141
|
+
* backend rather than chosen here:
|
|
142
|
+
*
|
|
143
|
+
* - **No ANN vector index.** Turso Database implements the vector *functions*
|
|
144
|
+
* but not `libsql_vector_idx()` or `vector_top_k()`, so `createTable()` skips
|
|
145
|
+
* the vector index and `search()` always runs the exact full scan. Search is
|
|
146
|
+
* therefore O(rows) on this backend.
|
|
147
|
+
* - **Transactions, not batches.** Turso's `batch()` is not atomic — a failing
|
|
148
|
+
* batch can leave earlier statements committed — so the index replacement
|
|
149
|
+
* runs inside an explicit `BEGIN IMMEDIATE` / `COMMIT` transaction, which is.
|
|
150
|
+
* This preserves the guarantee that a failed rebuild leaves the previous
|
|
151
|
+
* index intact.
|
|
152
|
+
*/
|
|
153
|
+
declare function tursoAdapter(database: TursoDatabase): DatabaseAdapter;
|
|
154
|
+
|
|
155
|
+
export { tursoAdapter };
|
|
156
|
+
export type { DatabaseAdapter, TursoDatabase, TursoStatement };
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
function tursoAdapter(database) {
|
|
2
|
+
assertTursoDatabase(database);
|
|
3
|
+
return {
|
|
4
|
+
libsqlSearchAdapter: true,
|
|
5
|
+
backend: "turso",
|
|
6
|
+
/**
|
|
7
|
+
* Turso Database ships the vector functions but no ANN index. This is a
|
|
8
|
+
* design direction rather than a gap: the project shipped SIMD-accelerated
|
|
9
|
+
* exact search instead, and the DiskANN port remains an open backlog item.
|
|
10
|
+
*/
|
|
11
|
+
supportsVectorIndex: false,
|
|
12
|
+
async executeDdl(sql) {
|
|
13
|
+
await database.exec(sql);
|
|
14
|
+
},
|
|
15
|
+
async executeQuery(sql, args) {
|
|
16
|
+
const statement = database.prepare(sql);
|
|
17
|
+
try {
|
|
18
|
+
const rows = await (args === void 0 ? statement.all() : statement.all(args));
|
|
19
|
+
return rows;
|
|
20
|
+
} finally {
|
|
21
|
+
closeStatement(statement);
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
/**
|
|
25
|
+
* Replace the table contents inside one explicit transaction.
|
|
26
|
+
*
|
|
27
|
+
* `batch()` is deliberately not used. It is not transactional on this
|
|
28
|
+
* backend: a batch that fails part way through leaves the statements before
|
|
29
|
+
* the failure applied, which would silently turn a failed rebuild into a
|
|
30
|
+
* half-destroyed index. `BEGIN IMMEDIATE` takes the write lock up front and
|
|
31
|
+
* `ROLLBACK` restores the previous rows exactly.
|
|
32
|
+
*
|
|
33
|
+
* Statements are prepared once per distinct SQL string and rebound per row.
|
|
34
|
+
* An index rebuild is one `DELETE` followed by N identical `INSERT`s, so
|
|
35
|
+
* this turns N+1 prepares into exactly 2 regardless of corpus size.
|
|
36
|
+
*/
|
|
37
|
+
async executeAtomicWrite(statements) {
|
|
38
|
+
const preparedBySql = /* @__PURE__ */ new Map();
|
|
39
|
+
const prepareOnce = (sql) => {
|
|
40
|
+
let prepared = preparedBySql.get(sql);
|
|
41
|
+
if (prepared === void 0) {
|
|
42
|
+
prepared = database.prepare(sql);
|
|
43
|
+
preparedBySql.set(sql, prepared);
|
|
44
|
+
}
|
|
45
|
+
return prepared;
|
|
46
|
+
};
|
|
47
|
+
await database.exec("BEGIN IMMEDIATE");
|
|
48
|
+
try {
|
|
49
|
+
for (const statement of statements) {
|
|
50
|
+
const prepared = prepareOnce(statement.sql);
|
|
51
|
+
await (statement.args === void 0 ? prepared.run() : prepared.run(statement.args));
|
|
52
|
+
}
|
|
53
|
+
await database.exec("COMMIT");
|
|
54
|
+
} catch (error) {
|
|
55
|
+
try {
|
|
56
|
+
await database.exec("ROLLBACK");
|
|
57
|
+
} catch (rollbackError) {
|
|
58
|
+
warnOnFailedRollback(rollbackError);
|
|
59
|
+
}
|
|
60
|
+
throw error;
|
|
61
|
+
} finally {
|
|
62
|
+
for (const prepared of preparedBySql.values()) {
|
|
63
|
+
closeStatement(prepared);
|
|
64
|
+
}
|
|
65
|
+
preparedBySql.clear();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
function closeStatement(statement) {
|
|
71
|
+
try {
|
|
72
|
+
statement.close?.();
|
|
73
|
+
} catch {
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const NO_ACTIVE_TRANSACTION_PATTERN = /no transaction is active/i;
|
|
77
|
+
function warnOnFailedRollback(error) {
|
|
78
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
79
|
+
if (NO_ACTIVE_TRANSACTION_PATTERN.test(message)) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
console.warn(
|
|
83
|
+
`[libsql-search] ROLLBACK failed after a failed index replacement: ${message}. This database handle may still be inside an open transaction, in which case a later write on it fails with "cannot start a transaction within a transaction". Reconnect the handle before reusing it.`
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
function assertTursoDatabase(database) {
|
|
87
|
+
if (typeof database !== "object" || database === null || typeof database.exec !== "function" || typeof database.prepare !== "function") {
|
|
88
|
+
throw new TypeError(
|
|
89
|
+
"tursoAdapter() expects a connected @tursodatabase/database handle with exec() and prepare() methods. connect() returns a Promise, so remember to await it."
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export { tursoAdapter };
|