@tinyshed/tinystore 0.4.1 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  // memory of the core's comes back: it reads our frames where they are and
5
5
  // writes its own where we say. Bun alone: bun:ffi.
6
6
 
7
- import { dlopen, FFIType, JSCallback, type Pointer, ptr } from 'bun:ffi'
7
+ import { dlopen, FFIType, JSCallback, type Library, type Pointer, ptr } from 'bun:ffi'
8
8
 
9
9
  import { ClosedError } from '../errors.ts'
10
10
  import type { Transport } from '../runtime.ts'
@@ -37,6 +37,8 @@ const symbols = {
37
37
  tinystore_close: { args: [FFIType.ptr], returns: FFIType.void },
38
38
  } as const
39
39
 
40
+ type Core = Library<typeof symbols>
41
+
40
42
  /** What the pipe tells the session reading it. */
41
43
  export interface PipeEvents {
42
44
  /**
@@ -65,144 +67,174 @@ export function openPipe(
65
67
  events: PipeEvents,
66
68
  ): Transport {
67
69
  const lib = dlopen(library, symbols)
70
+ const pipe = new CorePipe(lib, new Inbox(events))
71
+ const wake = new JSCallback(() => pipe.readOn(undefined), {
72
+ args: [FFIType.ptr],
73
+ returns: FFIType.void,
74
+ threadsafe: true,
75
+ })
76
+ pipe.connection = connect(lib, dir, options, wake)
77
+ // Bun's loop with no timer armed spins while it waits for a threadsafe
78
+ // callback from another thread: a write waiting for its sync burned a
79
+ // whole processor (research release-2026-10-23). A timer armed, however
80
+ // far off, lets the loop sleep until the core wakes it; it also keeps the
81
+ // process alive while the store is open, as a sidecar's socket does.
82
+ const sleeps = setInterval(() => {}, 2 ** 30)
83
+ return {
84
+ write: turning(bytes => pipe.send(bytes)),
85
+ close() {
86
+ const closing = pipe.connection
87
+ if (closing === null) {
88
+ return
89
+ }
90
+ pipe.connection = null
91
+ clearInterval(sleeps)
92
+ lib.symbols.tinystore_close(closing)
93
+ wake.close()
94
+ events.end(new ClosedError('the store closed'))
95
+ },
96
+ }
97
+ }
98
+
99
+ /** Opens the store through the core; a store that does not open lets the core and the wake go. */
100
+ function connect(lib: Core, dir: string, options: Uint8Array, wake: JSCallback): Pointer {
68
101
  const name = new TextEncoder().encode(dir)
69
102
  const opened = new BigUint64Array(1)
70
103
  const failure = new Uint8Array(1024)
71
- let connection: Pointer | null = null
104
+ const failed = Number(
105
+ lib.symbols.tinystore_open(
106
+ ptr(name),
107
+ name.length,
108
+ ptr(options),
109
+ options.length,
110
+ wake,
111
+ null,
112
+ ptr(opened),
113
+ ptr(failure),
114
+ failure.length,
115
+ ),
116
+ )
117
+ if (failed > 0) {
118
+ const message = new TextDecoder().decode(failure.subarray(0, failed))
119
+ wake.close()
120
+ lib.close()
121
+ throw openFailure(dir, message)
122
+ }
123
+ return Number(opened[0]) as Pointer
124
+ }
72
125
 
73
- // where the core writes what it owes: whole frames are read from the
74
- // front, and a frame's first part is held there for its rest
75
- const inbox = new Uint8Array(room)
76
- let held = 0
126
+ /**
127
+ * Where the core writes what it owes: whole frames are read from the front,
128
+ * and a frame's first part is held there for its rest.
129
+ */
130
+ class Inbox {
131
+ readonly #events: PipeEvents
132
+ readonly #inbox = new Uint8Array(room)
133
+ #held = 0
77
134
  // A frame longer than the inbox, in bytes as long as it is, and how much
78
135
  // of it came: the core writes its rest there, and the session keeps them,
79
136
  // so that a file's mebibyte is written once here rather than copied out
80
137
  // of an inbox grown for it (issue #10).
81
- let large: Uint8Array | undefined
82
- let filledLarge = 0
83
-
84
- // where the core writes next, and how many bytes it may
85
- const into = (): [Uint8Array, number, number] =>
86
- large === undefined
87
- ? [inbox, held, inbox.length - held]
88
- : [large, filledLarge, large.length - filledLarge]
89
-
90
- // reads what the core wrote where `into` said, and says whether it filled
91
- // that room, which means more may wait
92
- const read = (written: number): boolean => {
93
- if (large !== undefined) {
94
- filledLarge += written
95
- if (filledLarge < large.length) {
138
+ #large: Uint8Array | undefined
139
+ #filledLarge = 0
140
+
141
+ constructor(events: PipeEvents) {
142
+ this.#events = events
143
+ }
144
+
145
+ /** Where the core writes next, and how many bytes it may. */
146
+ into(): [Uint8Array, number, number] {
147
+ return this.#large === undefined
148
+ ? [this.#inbox, this.#held, this.#inbox.length - this.#held]
149
+ : [this.#large, this.#filledLarge, this.#large.length - this.#filledLarge]
150
+ }
151
+
152
+ /**
153
+ * Reads what the core wrote where `into` said, and says whether it filled
154
+ * that room, which means more may wait.
155
+ */
156
+ read(written: number): boolean {
157
+ if (this.#large !== undefined) {
158
+ this.#filledLarge += written
159
+ if (this.#filledLarge < this.#large.length) {
96
160
  return false
97
161
  }
98
- const whole = large
99
- large = undefined
100
- events.frame(whole)
162
+ const whole = this.#large
163
+ this.#large = undefined
164
+ this.#events.frame(whole)
101
165
  return true
102
166
  }
103
- const length = held + written
104
- const filled = length === inbox.length
105
- const taken = written === 0 ? 0 : events.frames(inbox, length)
106
- inbox.copyWithin(0, taken, length)
107
- held = length - taken
108
- const frameLength = held < headerSize ? 0 : headerSize + bodyLength(inbox)
109
- if (frameLength > inbox.length) {
110
- large = new Uint8Array(frameLength)
111
- large.set(inbox.subarray(0, held))
112
- filledLarge = held
113
- held = 0
167
+ const length = this.#held + written
168
+ const filled = length === this.#inbox.length
169
+ const taken = written === 0 ? 0 : this.#events.frames(this.#inbox, length)
170
+ this.#inbox.copyWithin(0, taken, length)
171
+ this.#held = length - taken
172
+ const frameLength = this.#held < headerSize ? 0 : headerSize + bodyLength(this.#inbox)
173
+ if (frameLength > this.#inbox.length) {
174
+ this.#large = new Uint8Array(frameLength)
175
+ this.#large.set(this.#inbox.subarray(0, this.#held))
176
+ this.#filledLarge = this.#held
177
+ this.#held = 0
114
178
  }
115
179
  return filled
116
180
  }
181
+ }
117
182
 
183
+ /** One connection's calls into the core, which write what it owes into the inbox. */
184
+ class CorePipe {
185
+ connection: Pointer | null = null
186
+ readonly #lib: Core
187
+ readonly #inbox: Inbox
118
188
  // What the session writes while it reads: its frames wait for the read's
119
189
  // end, since a call of the core made meanwhile would write over the bytes
120
190
  // being read.
121
- let reading = false
122
- const waiting: Uint8Array[] = []
123
-
124
- // reads what a call of the core wrote, and on while the core fills the
125
- // room, since no wake comes for what waits; then sends what was written
126
- // meanwhile
127
- const readOn = (written: number | undefined): void => {
128
- reading = true
191
+ #reading = false
192
+ readonly #waiting: Uint8Array[] = []
193
+
194
+ constructor(lib: Core, inbox: Inbox) {
195
+ this.#lib = lib
196
+ this.#inbox = inbox
197
+ }
198
+
199
+ /**
200
+ * Reads what a call of the core wrote, and on while the core fills the
201
+ * room, since no wake comes for what waits; then sends what was written
202
+ * meanwhile.
203
+ */
204
+ readOn(written: number | undefined): void {
205
+ this.#reading = true
129
206
  try {
130
- let filled = written === undefined || read(written)
131
- while (filled && connection !== null) {
132
- const [bytes, at, free] = into()
133
- filled = read(Number(lib.symbols.tinystore_recv(connection, 0, ptr(bytes, at), free)))
207
+ let filled = written === undefined || this.#inbox.read(written)
208
+ while (filled && this.connection !== null) {
209
+ const [bytes, at, free] = this.#inbox.into()
210
+ const read = this.#lib.symbols.tinystore_recv(this.connection, 0, ptr(bytes, at), free)
211
+ filled = this.#inbox.read(Number(read))
134
212
  }
135
213
  } finally {
136
- reading = false
214
+ this.#reading = false
137
215
  }
138
- const next = waiting.shift()
216
+ const next = this.#waiting.shift()
139
217
  if (next !== undefined) {
140
- send(next)
218
+ this.send(next)
141
219
  }
142
220
  }
143
221
 
144
- const send = (bytes: Uint8Array): void => {
145
- if (connection === null) {
222
+ send(bytes: Uint8Array): void {
223
+ if (this.connection === null) {
146
224
  return
147
225
  }
148
- if (reading) {
149
- waiting.push(bytes)
226
+ if (this.#reading) {
227
+ this.#waiting.push(bytes)
150
228
  return
151
229
  }
152
- const [owed, at, free] = into()
153
- const written = lib.symbols.tinystore_send(
154
- connection,
230
+ const [owed, at, free] = this.#inbox.into()
231
+ const written = this.#lib.symbols.tinystore_send(
232
+ this.connection,
155
233
  ptr(bytes),
156
234
  bytes.length,
157
235
  ptr(owed, at),
158
236
  free,
159
237
  )
160
- readOn(Number(written))
161
- }
162
-
163
- const wake = new JSCallback(() => readOn(undefined), {
164
- args: [FFIType.ptr],
165
- returns: FFIType.void,
166
- threadsafe: true,
167
- })
168
- const failed = Number(
169
- lib.symbols.tinystore_open(
170
- ptr(name),
171
- name.length,
172
- ptr(options),
173
- options.length,
174
- wake,
175
- null,
176
- ptr(opened),
177
- ptr(failure),
178
- failure.length,
179
- ),
180
- )
181
- if (failed > 0) {
182
- const message = new TextDecoder().decode(failure.subarray(0, failed))
183
- wake.close()
184
- lib.close()
185
- throw openFailure(dir, message)
186
- }
187
- connection = Number(opened[0]) as Pointer
188
- // Bun's loop with no timer armed spins while it waits for a threadsafe
189
- // callback from another thread: a write waiting for its sync burned a
190
- // whole processor (research release-2026-10-23). A timer armed, however
191
- // far off, lets the loop sleep until the core wakes it; it also keeps the
192
- // process alive while the store is open, as a sidecar's socket does.
193
- const sleeps = setInterval(() => {}, 2 ** 30)
194
- return {
195
- write: turning(send),
196
- close() {
197
- if (connection === null) {
198
- return
199
- }
200
- const closing = connection
201
- connection = null
202
- clearInterval(sleeps)
203
- lib.symbols.tinystore_close(closing)
204
- wake.close()
205
- events.end(new ClosedError('the store closed'))
206
- },
238
+ this.readOn(Number(written))
207
239
  }
208
240
  }
package/src/search.ts CHANGED
@@ -38,6 +38,12 @@ export interface IndexOptions {
38
38
  filters?: readonly string[] | undefined
39
39
  /** a misspelt word read as the index's nearest; on unless false */
40
40
  typoTolerance?: boolean | undefined
41
+ /**
42
+ * the keyboard layouts the index's readers type with, `['ru', 'en']`: a
43
+ * word that finds nothing as typed is read as the same keys type in the
44
+ * other layout, when it finds something there, and `corrected` says so
45
+ */
46
+ layouts?: readonly KeyboardLayout[] | undefined
41
47
  /** a vector the program's model gives each document, and `recall` for searches from clusters */
42
48
  vectors?: { dimensions: number; recall?: number | undefined } | undefined
43
49
  /** how a search by words and a vector at once combines them unless it says */
@@ -46,6 +52,9 @@ export interface IndexOptions {
46
52
  rebuild?: 'automatic' | 'manual' | undefined
47
53
  }
48
54
 
55
+ /** A keyboard layout an index's readers type with: ЙЦУКЕН, or QWERTY. */
56
+ export type KeyboardLayout = 'ru' | 'en'
57
+
49
58
  /** A vector the program's own model gave, of the index's dimensions. */
50
59
  export type Vector = readonly number[] | Float32Array
51
60
 
@@ -88,19 +97,46 @@ export interface Query {
88
97
  /**
89
98
  * the last word read as the start of a word too, as a search box types:
90
99
  * `crypt` finds `cryptography` as well, at half its field's weight; a
91
- * word a space follows is whole, and so is one of a single letter
100
+ * word a space follows is whole, and so is one of a single letter. With
101
+ * `'every'`, every word of two letters or more is read so, the first
102
+ * eight, as people shorten each word of a name: `лин алг` finds
103
+ * `Линейная алгебра`; a document that holds a word for each word of the
104
+ * text comes first, its score raised above the rest
105
+ */
106
+ prefix?: boolean | 'every' | undefined
107
+ /**
108
+ * each hit says where the search's words matched it, in `matches`, as a
109
+ * search box shows them in bold: a word found as written, by its stem or
110
+ * once corrected whole, and a word a prefix began by the letters typed
92
111
  */
93
- prefix?: boolean | undefined
112
+ matches?: boolean | undefined
94
113
  /** 10 unless said; offset and limit together 1000 at most */
95
114
  limit?: number | undefined
96
115
  offset?: number | undefined
97
116
  }
98
117
 
118
+ /**
119
+ * Where a search's word matched a hit: `start` to `end` of a field's text in
120
+ * UTF-16 units, so that `text.slice(start, end)` is what matched.
121
+ */
122
+ export interface Match {
123
+ /** the field's path, or a table's column; `''` for a document that is its own text */
124
+ field: string
125
+ /** the string's place in a field that holds a list */
126
+ item?: number
127
+ start: number
128
+ end: number
129
+ /** the query's word, as its text wrote it */
130
+ word: string
131
+ }
132
+
99
133
  /** A document a search found, and the score that ordered it among this search's hits. */
100
134
  export interface Hit<T> {
101
135
  id: string
102
136
  value: T
103
137
  score: number
138
+ /** where the search's words matched it, when the search asked with `matches` */
139
+ matches?: Match[]
104
140
  }
105
141
 
106
142
  /** What a search found, the best first, and the text it read in fact when a word was misspelt. */
@@ -139,6 +175,7 @@ export interface TableIndexOptions {
139
175
  /** the columns a search may keep its hits to; a boolean is the 1 or 0 SQLite keeps */
140
176
  filters?: readonly string[] | undefined
141
177
  typoTolerance?: boolean | undefined
178
+ layouts?: readonly KeyboardLayout[] | undefined
142
179
  /** a column of blobs of little-endian floats, one a row or null, and `recall` for searches from clusters */
143
180
  vectors?: { column: string; dimensions: number; recall?: number | undefined } | undefined
144
181
  hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
@@ -193,20 +230,29 @@ abstract class Searchable<T> {
193
230
  vector,
194
231
  vectorWeight: asked.hybrid?.vectorWeight,
195
232
  rerank: asked.hybrid?.rerank,
196
- prefix: asked.prefix,
233
+ prefix: asked.prefix === true || undefined,
234
+ // none unless asked, so that a server before it reads the search
235
+ prefixEvery: asked.prefix === 'every' || undefined,
236
+ // JavaScript slices a string by its UTF-16 units
237
+ matches: asked.matches === true ? 'utf16' : undefined,
197
238
  where,
198
239
  offset: asked.offset,
199
240
  limit: asked.limit,
200
241
  })
201
242
  const found = SearchFound.decode(await this.call('search.search', body, 'read'))
202
243
  const columns = found.columns
203
- const hits = (found.hits ?? []).map(hit => ({
204
- id: hit.id ?? '',
205
- value: (columns === undefined
206
- ? parseJson(hit.value ?? 'null')
207
- : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]) as T,
208
- score: hit.score ?? 0,
209
- }))
244
+ const hits = (found.hits ?? []).map(hit => {
245
+ const value = (
246
+ columns === undefined
247
+ ? parseJson(hit.value ?? 'null')
248
+ : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]
249
+ ) as T
250
+ const read: Hit<T> = { id: hit.id ?? '', value, score: hit.score ?? 0 }
251
+ if (hit.matches !== undefined) {
252
+ read.matches = hit.matches.map(matchOf)
253
+ }
254
+ return read
255
+ })
210
256
  return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected }
211
257
  }
212
258
 
@@ -261,7 +307,12 @@ export class SearchIndex<T> extends Searchable<T> {
261
307
  )
262
308
  }
263
309
 
264
- /** Sets every document in one commit, all of them or none: 1,000 and 16 MiB at most. */
310
+ /**
311
+ * Sets every document in one commit, all of them or none: 1,000 at most,
312
+ * `limits.setManyDocuments`, and no more than one request body holds,
313
+ * 2 MiB, `limits.requestBytes`. Past either it fails with a LimitError
314
+ * that names the bound.
315
+ */
265
316
  async setMany(documents: readonly IndexDocument<T>[]): Promise<void> {
266
317
  const encoded = documents.map(document => ({
267
318
  id: keyText(document.id),
@@ -318,6 +369,7 @@ export function tableIndexOpen(
318
369
  fields: fieldsOf(options.fields),
319
370
  filters: [...(options.filters ?? [])],
320
371
  typoTolerance: options.typoTolerance,
372
+ layouts: layoutsOf(options.layouts),
321
373
  dimensions: options.vectors?.dimensions,
322
374
  recall: options.vectors?.recall,
323
375
  vectorWeight: options.hybrid?.vectorWeight,
@@ -337,6 +389,11 @@ function fieldsOf(fields: IndexOptions['fields']): { path: string; weight: numbe
337
389
  : Object.entries(fields ?? {}).map(([path, weight]) => ({ path, weight }))
338
390
  }
339
391
 
392
+ /** The layouts asked for; none left out, so that a server before them reads the open. */
393
+ function layoutsOf(layouts: readonly KeyboardLayout[] | undefined): KeyboardLayout[] | undefined {
394
+ return layouts?.length ? [...layouts] : undefined
395
+ }
396
+
340
397
  /** The open of an index: its name, settings, and whether it reads the index again. */
341
398
  export function indexOpen(name: string, options: IndexOptions = {}, reindex = false): Uint8Array {
342
399
  checkName(name, 'index')
@@ -345,6 +402,7 @@ export function indexOpen(name: string, options: IndexOptions = {}, reindex = fa
345
402
  fields: fieldsOf(options.fields),
346
403
  filters: [...(options.filters ?? [])],
347
404
  typoTolerance: options.typoTolerance,
405
+ layouts: layoutsOf(options.layouts),
348
406
  dimensions: options.vectors?.dimensions,
349
407
  recall: options.vectors?.recall,
350
408
  vectorWeight: options.hybrid?.vectorWeight,
@@ -397,6 +455,25 @@ function statusOf(status: {
397
455
  }
398
456
  }
399
457
 
458
+ function matchOf(found: {
459
+ field?: string | undefined
460
+ item?: number | undefined
461
+ start?: number | undefined
462
+ end?: number | undefined
463
+ word?: string | undefined
464
+ }): Match {
465
+ const match: Match = {
466
+ field: found.field ?? '',
467
+ start: found.start ?? 0,
468
+ end: found.end ?? 0,
469
+ word: found.word ?? '',
470
+ }
471
+ if (found.item !== undefined) {
472
+ match.item = found.item
473
+ }
474
+ return match
475
+ }
476
+
400
477
  function documentJson(name: string, value: unknown): string {
401
478
  const json = JSON.stringify(value)
402
479
  if (json === undefined) {
package/src/session.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  ClosedError,
13
13
  errorOf,
14
14
  LimitError,
15
+ limits,
15
16
  ProtocolError,
16
17
  type TinystoreError,
17
18
  UnavailableError,
@@ -31,10 +32,20 @@ import {
31
32
  putHeader,
32
33
  } from './wire/frame.ts'
33
34
  import { Reader, Writer } from './wire/msgpack.ts'
34
- import { Failure, GoAway, Hello, Welcome } from './wire/protocol.ts'
35
+ import { Failure, GoAway, Hello, methods, Welcome } from './wire/protocol.ts'
35
36
 
36
37
  export const protocol = 1
37
38
 
39
+ /** Each method's name by its number, as the schema spells it: `search.setMany`. */
40
+ const methodNames = new Map<number, string>(
41
+ Object.entries(methods).map(([name, number]) => [number, name]),
42
+ )
43
+
44
+ /** A method's name as the schema spells it, for an error the SDK says itself. */
45
+ function methodName(method: number): string {
46
+ return methodNames.get(method) ?? `method ${method}`
47
+ }
48
+
38
49
  /** What this client lets the server send on a stream before it grants more: a body at least. */
39
50
  export const downloadWindow = 2 << 20
40
51
 
@@ -365,7 +376,12 @@ export class Session {
365
376
  const agreed = await unlessCancelled(this.welcomed, signal)
366
377
  if (body.length > agreed.maxBody) {
367
378
  throw new LimitError(
368
- `a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
379
+ `${methodName(method)}: a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
380
+ {
381
+ limit: limits.requestBytes.name,
382
+ wanted: String(body.length),
383
+ bound: String(agreed.maxBody),
384
+ },
369
385
  )
370
386
  }
371
387
  while (this.#streams.size >= agreed.inFlight && this.#usable() === undefined) {