@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/src/search.ts
ADDED
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
// Search, as docs/guides/search/README.md has it: documents of one type set by id in a
|
|
2
|
+
// named index and found again by their words, by vectors the program's own
|
|
3
|
+
// model gives them, or by both, kept to the documents a `where` passes. A
|
|
4
|
+
// search answers the exact best ten, or `limit`, of what the index holds; an
|
|
5
|
+
// index opened with `recall` answers its vectors from clusters. A table's
|
|
6
|
+
// index, `db.searchIndex`, finds a database's rows, each hit its row.
|
|
7
|
+
|
|
8
|
+
import type { Link } from './connection.ts'
|
|
9
|
+
import { InvalidError } from './errors.ts'
|
|
10
|
+
import { checkName, type Open, type Via, viaLink } from './handles.ts'
|
|
11
|
+
import { parseJson } from './json.ts'
|
|
12
|
+
import { keyText } from './kv.ts'
|
|
13
|
+
import { rowsOf } from './sql.ts'
|
|
14
|
+
import type { Key, SqlValue } from './wire/codec.ts'
|
|
15
|
+
import {
|
|
16
|
+
Empty,
|
|
17
|
+
Handle,
|
|
18
|
+
methods,
|
|
19
|
+
SearchDeleted,
|
|
20
|
+
SearchDropped,
|
|
21
|
+
SearchFound,
|
|
22
|
+
SearchGot,
|
|
23
|
+
SearchKey,
|
|
24
|
+
SearchListed,
|
|
25
|
+
SearchName,
|
|
26
|
+
SearchOpen,
|
|
27
|
+
SearchQuery,
|
|
28
|
+
SearchSet,
|
|
29
|
+
SearchSetMany,
|
|
30
|
+
SearchStatus,
|
|
31
|
+
} from './wire/protocol.ts'
|
|
32
|
+
|
|
33
|
+
/** How an index reads its documents and combines its halves, given where it opens. */
|
|
34
|
+
export interface IndexOptions {
|
|
35
|
+
/** the fields whose words are read, each with what a word found there weighs, or a list that weighs each 1 */
|
|
36
|
+
fields?: Record<string, number> | readonly string[] | undefined
|
|
37
|
+
/** the fields a search may keep its hits to */
|
|
38
|
+
filters?: readonly string[] | undefined
|
|
39
|
+
/** a misspelt word read as the index's nearest; on unless false */
|
|
40
|
+
typoTolerance?: boolean | undefined
|
|
41
|
+
/** a vector the program's model gives each document, and `recall` for searches from clusters */
|
|
42
|
+
vectors?: { dimensions: number; recall?: number | undefined } | undefined
|
|
43
|
+
/** how a search by words and a vector at once combines them unless it says */
|
|
44
|
+
hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
|
|
45
|
+
/** whether a change that needs every document read again waits for `store.reindex` */
|
|
46
|
+
rebuild?: 'automatic' | 'manual' | undefined
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** A vector the program's own model gave, of the index's dimensions. */
|
|
50
|
+
export type Vector = readonly number[] | Float32Array
|
|
51
|
+
|
|
52
|
+
/** A value a `where` matches: text, a number, a boolean, one of a list, a range, or `null` for none. */
|
|
53
|
+
export type WhereValue =
|
|
54
|
+
| string
|
|
55
|
+
| number
|
|
56
|
+
| bigint
|
|
57
|
+
| boolean
|
|
58
|
+
| null
|
|
59
|
+
| readonly (string | number | bigint | boolean)[]
|
|
60
|
+
| { gt?: number | bigint; gte?: number | bigint; lt?: number | bigint; lte?: number | bigint }
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* What a search keeps its hits to, as sqldb's `where` reads an object: each
|
|
64
|
+
* field equal to its value, one of a list, within a range, or `null` for
|
|
65
|
+
* none, every field named passing; `search.or` and `search.and` group them.
|
|
66
|
+
*/
|
|
67
|
+
export type Where = Readonly<Record<string, WhereValue>> | Group
|
|
68
|
+
|
|
69
|
+
/** Conditions joined: `search.or(…)`, `search.and(…)`. */
|
|
70
|
+
export class Group {
|
|
71
|
+
readonly join: 'or' | 'and'
|
|
72
|
+
readonly parts: readonly Where[]
|
|
73
|
+
|
|
74
|
+
/** A group comes from `search.or` and `search.and`. */
|
|
75
|
+
constructor(join: 'or' | 'and', parts: readonly Where[]) {
|
|
76
|
+
this.join = join
|
|
77
|
+
this.parts = parts
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** What a search asks: its words, its vector, or both, and what it keeps to. */
|
|
82
|
+
export interface Query {
|
|
83
|
+
text?: string | undefined
|
|
84
|
+
vector?: Vector | undefined
|
|
85
|
+
where?: Where | undefined
|
|
86
|
+
/** this search's way of combining its words and its vector, in place of the index's */
|
|
87
|
+
hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
|
|
88
|
+
/** 10 unless said; offset and limit together 1000 at most */
|
|
89
|
+
limit?: number | undefined
|
|
90
|
+
offset?: number | undefined
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** A document a search found, and the score that ordered it among this search's hits. */
|
|
94
|
+
export interface Hit<T> {
|
|
95
|
+
id: string
|
|
96
|
+
value: T
|
|
97
|
+
score: number
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** What a search found, the best first, and the text it read in fact when a word was misspelt. */
|
|
101
|
+
export interface Found<T> {
|
|
102
|
+
hits: Hit<T>[]
|
|
103
|
+
corrected?: string | undefined
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Where an index stands: serving what it was opened with, read again beside it, or a rebuild that failed. */
|
|
107
|
+
export type IndexStatus =
|
|
108
|
+
| { state: 'ready' }
|
|
109
|
+
| { state: 'rebuilding'; done: number }
|
|
110
|
+
| { state: 'failed'; error: string }
|
|
111
|
+
|
|
112
|
+
/** An index as `store.indexes()` lists it. */
|
|
113
|
+
export interface IndexInfo {
|
|
114
|
+
name: string
|
|
115
|
+
documents: number
|
|
116
|
+
vectors: number
|
|
117
|
+
/** when a program last opened it */
|
|
118
|
+
opened?: Date | undefined
|
|
119
|
+
status: IndexStatus
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* How a table's index reads a table of its database: its columns of text,
|
|
124
|
+
* those a search may filter by, and a column of vectors the program fills.
|
|
125
|
+
*/
|
|
126
|
+
export interface TableIndexOptions {
|
|
127
|
+
/** the table whose rows the index holds and follows */
|
|
128
|
+
table: string
|
|
129
|
+
/** the column of unique values a row is found by, the table's primary key unless said */
|
|
130
|
+
key?: string | undefined
|
|
131
|
+
/** the columns whose words are read, each with what a word found there weighs, or a list that weighs each 1 */
|
|
132
|
+
fields: Record<string, number> | readonly string[]
|
|
133
|
+
/** the columns a search may keep its hits to; a boolean is the 1 or 0 SQLite keeps */
|
|
134
|
+
filters?: readonly string[] | undefined
|
|
135
|
+
typoTolerance?: boolean | undefined
|
|
136
|
+
/** a column of blobs of little-endian floats, one a row or null, and `recall` for searches from clusters */
|
|
137
|
+
vectors?: { column: string; dimensions: number; recall?: number | undefined } | undefined
|
|
138
|
+
hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
|
|
139
|
+
rebuild?: 'automatic' | 'manual' | undefined
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** A document `setMany` sets: its id, its value, and its vector, or `null` for none. */
|
|
143
|
+
export interface IndexDocument<T> {
|
|
144
|
+
id: Key
|
|
145
|
+
value: T
|
|
146
|
+
vector?: Vector | null | undefined
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** How often `ready` asks whether a rebuild switched in. */
|
|
150
|
+
const readyEvery = 100
|
|
151
|
+
|
|
152
|
+
/** The calls of an index that a call names. */
|
|
153
|
+
type Method =
|
|
154
|
+
| 'search.set'
|
|
155
|
+
| 'search.setMany'
|
|
156
|
+
| 'search.delete'
|
|
157
|
+
| 'search.get'
|
|
158
|
+
| 'search.search'
|
|
159
|
+
| 'search.status'
|
|
160
|
+
|
|
161
|
+
/** What every index answers: its searches, and where it stands. */
|
|
162
|
+
abstract class Searchable<T> {
|
|
163
|
+
readonly name: string
|
|
164
|
+
readonly #via: Via
|
|
165
|
+
readonly #open: Open
|
|
166
|
+
|
|
167
|
+
constructor(via: Via, name: string, open: Open) {
|
|
168
|
+
this.name = name
|
|
169
|
+
this.#via = via
|
|
170
|
+
this.#open = open
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* The best ten, or `limit`, by `text`'s words, a string being a search
|
|
175
|
+
* by its words; by a vector; or by both, each half's best hundred scaled
|
|
176
|
+
* and summed at the index's `vectorWeight`; kept to what `where` passes.
|
|
177
|
+
* Of equal scores the one set first comes first.
|
|
178
|
+
*/
|
|
179
|
+
async search(query: string | Query): Promise<Found<T>> {
|
|
180
|
+
const asked = typeof query === 'string' ? { text: query } : query
|
|
181
|
+
const vector = asked.vector === undefined ? undefined : vectorBytes(asked.vector)
|
|
182
|
+
const where = asked.where === undefined ? undefined : whereJson(asked.where)
|
|
183
|
+
const body = (handle: number) =>
|
|
184
|
+
SearchQuery.encode({
|
|
185
|
+
handle,
|
|
186
|
+
text: asked.text ?? '',
|
|
187
|
+
vector,
|
|
188
|
+
vectorWeight: asked.hybrid?.vectorWeight,
|
|
189
|
+
rerank: asked.hybrid?.rerank,
|
|
190
|
+
where,
|
|
191
|
+
offset: asked.offset,
|
|
192
|
+
limit: asked.limit,
|
|
193
|
+
})
|
|
194
|
+
const found = SearchFound.decode(await this.call('search.search', body, 'read'))
|
|
195
|
+
const columns = found.columns
|
|
196
|
+
const hits = (found.hits ?? []).map(hit => ({
|
|
197
|
+
id: hit.id ?? '',
|
|
198
|
+
value: (columns === undefined
|
|
199
|
+
? parseJson(hit.value ?? 'null')
|
|
200
|
+
: rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]) as T,
|
|
201
|
+
score: hit.score ?? 0,
|
|
202
|
+
}))
|
|
203
|
+
return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Where the index stands; a search never waits, and answers by the settings it serves. */
|
|
207
|
+
async status(): Promise<IndexStatus> {
|
|
208
|
+
const answer = await this.call('search.status', handle => Handle.encode({ handle }), 'read')
|
|
209
|
+
return statusOf(SearchStatus.decode(answer))
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Waits until the index serves the settings it was opened with; a rebuild that failed throws why. */
|
|
213
|
+
async ready(): Promise<void> {
|
|
214
|
+
for (;;) {
|
|
215
|
+
const status = await this.status()
|
|
216
|
+
if (status.state === 'ready') {
|
|
217
|
+
return
|
|
218
|
+
}
|
|
219
|
+
if (status.state === 'failed') {
|
|
220
|
+
throw new InvalidError(`index ${this.name}: its rebuild failed: ${status.error}`)
|
|
221
|
+
}
|
|
222
|
+
await new Promise(resolve => setTimeout(resolve, readyEvery))
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
protected call(
|
|
227
|
+
method: Method,
|
|
228
|
+
body: (handle: number) => Uint8Array,
|
|
229
|
+
idempotence: 'read' | 'write',
|
|
230
|
+
): Promise<Uint8Array> {
|
|
231
|
+
return this.#via.call(methods['search.open'], this.#open, methods[method], body, idempotence)
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Documents of one type by id, found again by their words, vectors or both: `store.index(name, options)`. */
|
|
236
|
+
export class SearchIndex<T> extends Searchable<T> {
|
|
237
|
+
/** An index comes from `store.index` and `store.reindex`. */
|
|
238
|
+
constructor(link: Link, name: string, open: Uint8Array) {
|
|
239
|
+
super(viaLink(link), name, open)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Sets the document `id` to `value`, replacing it whole; the next search
|
|
244
|
+
* sees it. On an index with vectors a set gives its vector, or `null` for
|
|
245
|
+
* a document only its words find.
|
|
246
|
+
*/
|
|
247
|
+
async set(id: Key, value: T, options?: { vector?: Vector | null | undefined }): Promise<void> {
|
|
248
|
+
const said = vectorSaid(options?.vector)
|
|
249
|
+
const json = documentJson(this.name, value)
|
|
250
|
+
await this.call(
|
|
251
|
+
'search.set',
|
|
252
|
+
handle => SearchSet.encode({ handle, id: keyText(id), value: json, ...said }),
|
|
253
|
+
'write',
|
|
254
|
+
)
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Sets every document in one commit, all of them or none: 1,000 and 16 MiB at most. */
|
|
258
|
+
async setMany(documents: readonly IndexDocument<T>[]): Promise<void> {
|
|
259
|
+
const encoded = documents.map(document => ({
|
|
260
|
+
id: keyText(document.id),
|
|
261
|
+
value: documentJson(this.name, document.value),
|
|
262
|
+
...vectorSaid(document.vector),
|
|
263
|
+
}))
|
|
264
|
+
await this.call(
|
|
265
|
+
'search.setMany',
|
|
266
|
+
handle => SearchSetMany.encode({ handle, documents: encoded }),
|
|
267
|
+
'write',
|
|
268
|
+
)
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Deletes the document `id`; says whether there was one. */
|
|
272
|
+
async delete(id: Key): Promise<boolean> {
|
|
273
|
+
const answer = await this.call(
|
|
274
|
+
'search.delete',
|
|
275
|
+
handle => SearchKey.encode({ handle, id: keyText(id) }),
|
|
276
|
+
'write',
|
|
277
|
+
)
|
|
278
|
+
return SearchDeleted.decode(answer).deleted ?? false
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** The document `id`, or undefined. */
|
|
282
|
+
async get(id: Key): Promise<T | undefined> {
|
|
283
|
+
const answer = await this.call(
|
|
284
|
+
'search.get',
|
|
285
|
+
handle => SearchKey.encode({ handle, id: keyText(id) }),
|
|
286
|
+
'read',
|
|
287
|
+
)
|
|
288
|
+
const got = SearchGot.decode(answer)
|
|
289
|
+
return got.found && got.value !== undefined ? (parseJson(got.value) as T) : undefined
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* A table's rows found again by their words, vectors or both, each hit's
|
|
295
|
+
* value its row as `db.all` reads it: `db.searchIndex(name, options)`. The
|
|
296
|
+
* table owns the rows, and every change of one reaches the index in the
|
|
297
|
+
* commit that makes it.
|
|
298
|
+
*/
|
|
299
|
+
export class TableIndex<T> extends Searchable<T> {}
|
|
300
|
+
|
|
301
|
+
/** The open of a table's index: its name, its table and settings, and its database's handle. */
|
|
302
|
+
export function tableIndexOpen(
|
|
303
|
+
name: string,
|
|
304
|
+
options: TableIndexOptions,
|
|
305
|
+
database?: number,
|
|
306
|
+
reindex = false,
|
|
307
|
+
): Uint8Array {
|
|
308
|
+
checkName(name, 'index')
|
|
309
|
+
return SearchOpen.encode({
|
|
310
|
+
name,
|
|
311
|
+
fields: fieldsOf(options.fields),
|
|
312
|
+
filters: [...(options.filters ?? [])],
|
|
313
|
+
typoTolerance: options.typoTolerance,
|
|
314
|
+
dimensions: options.vectors?.dimensions,
|
|
315
|
+
recall: options.vectors?.recall,
|
|
316
|
+
vectorWeight: options.hybrid?.vectorWeight,
|
|
317
|
+
rerank: options.hybrid?.rerank,
|
|
318
|
+
manual: options.rebuild === 'manual',
|
|
319
|
+
reindex,
|
|
320
|
+
database,
|
|
321
|
+
table: options.table,
|
|
322
|
+
key: options.key,
|
|
323
|
+
vectorColumn: options.vectors?.column,
|
|
324
|
+
})
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function fieldsOf(fields: IndexOptions['fields']): { path: string; weight: number }[] {
|
|
328
|
+
return Array.isArray(fields)
|
|
329
|
+
? fields.map(path => ({ path, weight: 1 }))
|
|
330
|
+
: Object.entries(fields ?? {}).map(([path, weight]) => ({ path, weight }))
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** The open of an index: its name, settings, and whether it reads the index again. */
|
|
334
|
+
export function indexOpen(name: string, options: IndexOptions = {}, reindex = false): Uint8Array {
|
|
335
|
+
checkName(name, 'index')
|
|
336
|
+
return SearchOpen.encode({
|
|
337
|
+
name,
|
|
338
|
+
fields: fieldsOf(options.fields),
|
|
339
|
+
filters: [...(options.filters ?? [])],
|
|
340
|
+
typoTolerance: options.typoTolerance,
|
|
341
|
+
dimensions: options.vectors?.dimensions,
|
|
342
|
+
recall: options.vectors?.recall,
|
|
343
|
+
vectorWeight: options.hybrid?.vectorWeight,
|
|
344
|
+
rerank: options.hybrid?.rerank,
|
|
345
|
+
manual: options.rebuild === 'manual',
|
|
346
|
+
reindex,
|
|
347
|
+
})
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** Drops an index and every document it holds; says whether there was one. */
|
|
351
|
+
export async function dropIndex(link: Link, name: string): Promise<boolean> {
|
|
352
|
+
checkName(name, 'index')
|
|
353
|
+
const answer = await link.run('write', connection =>
|
|
354
|
+
connection.session.call(methods['search.dropIndex'], SearchName.encode({ name })),
|
|
355
|
+
)
|
|
356
|
+
return SearchDropped.decode(answer).dropped ?? false
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** Every index the store holds, by name. */
|
|
360
|
+
export async function listIndexes(link: Link): Promise<IndexInfo[]> {
|
|
361
|
+
const answer = await link.run('read', connection =>
|
|
362
|
+
connection.session.call(methods['search.list'], Empty.encode({})),
|
|
363
|
+
)
|
|
364
|
+
return (SearchListed.decode(answer).indexes ?? []).map(index => {
|
|
365
|
+
const info: IndexInfo = {
|
|
366
|
+
name: index.name ?? '',
|
|
367
|
+
documents: index.documents ?? 0,
|
|
368
|
+
vectors: index.vectors ?? 0,
|
|
369
|
+
status: statusOf(index),
|
|
370
|
+
}
|
|
371
|
+
if (index.opened !== undefined) {
|
|
372
|
+
info.opened = new Date(index.opened)
|
|
373
|
+
}
|
|
374
|
+
return info
|
|
375
|
+
})
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function statusOf(status: {
|
|
379
|
+
state?: string | undefined
|
|
380
|
+
done?: number | undefined
|
|
381
|
+
error?: string | undefined
|
|
382
|
+
}): IndexStatus {
|
|
383
|
+
switch (status.state) {
|
|
384
|
+
case 'rebuilding':
|
|
385
|
+
return { state: 'rebuilding', done: status.done ?? 0 }
|
|
386
|
+
case 'failed':
|
|
387
|
+
return { state: 'failed', error: status.error ?? '' }
|
|
388
|
+
default:
|
|
389
|
+
return { state: 'ready' }
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function documentJson(name: string, value: unknown): string {
|
|
394
|
+
const json = JSON.stringify(value)
|
|
395
|
+
if (json === undefined) {
|
|
396
|
+
throw new InvalidError(`index ${name}: a document that does not write as JSON`)
|
|
397
|
+
}
|
|
398
|
+
return json
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** What a set says of its vector: its bytes, none, or nothing. */
|
|
402
|
+
function vectorSaid(vector: Vector | null | undefined): {
|
|
403
|
+
vector?: Uint8Array
|
|
404
|
+
noVector?: boolean
|
|
405
|
+
} {
|
|
406
|
+
if (vector === undefined) {
|
|
407
|
+
return {}
|
|
408
|
+
}
|
|
409
|
+
return vector === null ? { noVector: true } : { vector: vectorBytes(vector) }
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** A vector's floats' bytes, little-endian, four a dimension. */
|
|
413
|
+
function vectorBytes(vector: Vector): Uint8Array {
|
|
414
|
+
const floats = vector instanceof Float32Array ? vector : Float32Array.from(vector)
|
|
415
|
+
const bytes = new Uint8Array(floats.length * 4)
|
|
416
|
+
const view = new DataView(bytes.buffer)
|
|
417
|
+
for (const [at, value] of floats.entries()) {
|
|
418
|
+
view.setFloat32(at * 4, value, true)
|
|
419
|
+
}
|
|
420
|
+
return bytes
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* A `where` as the wire takes it, a JSON list of conditions joined by and:
|
|
425
|
+
* `{"eq":[path,value]}`, `{"in":[path,[values]]}`, `{"missing":path}`,
|
|
426
|
+
* `{"range":[path,{"gt"|"gte"|"lt"|"lte":n}]}`, `{"or":[…]}`, `{"and":[…]}`.
|
|
427
|
+
*/
|
|
428
|
+
export function whereJson(where: Where): string {
|
|
429
|
+
return `[${conditions(where).join(',')}]`
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function conditions(where: Where): string[] {
|
|
433
|
+
if (where instanceof Group) {
|
|
434
|
+
const parts = where.parts.map(part => `{"and":[${conditions(part).join(',')}]}`)
|
|
435
|
+
return [`{"${where.join}":[${parts.join(',')}]}`]
|
|
436
|
+
}
|
|
437
|
+
return Object.entries(where).map(([path, value]) => condition(path, value))
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function condition(path: string, value: WhereValue | undefined): string {
|
|
441
|
+
const at = JSON.stringify(path)
|
|
442
|
+
if (value === undefined) {
|
|
443
|
+
throw new InvalidError(
|
|
444
|
+
`a where of ${path}: undefined; leave ${path} out to ask nothing of it, or null for no value`,
|
|
445
|
+
)
|
|
446
|
+
}
|
|
447
|
+
if (value === null) {
|
|
448
|
+
return `{"missing":${at}}`
|
|
449
|
+
}
|
|
450
|
+
if (Array.isArray(value)) {
|
|
451
|
+
return `{"in":[${at},[${(value as readonly (string | number | bigint | boolean)[]).map(scalar).join(',')}]]}`
|
|
452
|
+
}
|
|
453
|
+
if (typeof value === 'object') {
|
|
454
|
+
const bounds = Object.entries(value).map(([bound, n]) => {
|
|
455
|
+
if (!['gt', 'gte', 'lt', 'lte'].includes(bound)) {
|
|
456
|
+
throw new InvalidError(`a where of ${path}: ${bound}, not gt, gte, lt or lte`)
|
|
457
|
+
}
|
|
458
|
+
return `${JSON.stringify(bound)}:${scalar(n as number | bigint)}`
|
|
459
|
+
})
|
|
460
|
+
return `{"range":[${at},{${bounds.join(',')}}]}`
|
|
461
|
+
}
|
|
462
|
+
return `{"eq":[${at},${scalar(value)}]}`
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** A value as JSON, a bigint as its digits so that an id of 19 digits matches as it is. */
|
|
466
|
+
function scalar(value: string | number | bigint | boolean): string {
|
|
467
|
+
if (typeof value === 'bigint') {
|
|
468
|
+
return value.toString()
|
|
469
|
+
}
|
|
470
|
+
if (typeof value === 'number' && !Number.isFinite(value)) {
|
|
471
|
+
throw new InvalidError(`a where's value ${value}, which is no number JSON writes`)
|
|
472
|
+
}
|
|
473
|
+
return JSON.stringify(value)
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/** The conditions a `where` groups: `search.or({ category: 'boots' }, { tags: 'sale' })`. */
|
|
477
|
+
export const search = {
|
|
478
|
+
or: (...parts: Where[]): Group => new Group('or', parts),
|
|
479
|
+
and: (...parts: Where[]): Group => new Group('and', parts),
|
|
480
|
+
}
|