@tinyshed/tinystore 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/errors.ts CHANGED
@@ -98,9 +98,10 @@ export class InvalidError extends TinystoreError {
98
98
  }
99
99
 
100
100
  /**
101
- * A bound: memory, size or count. Later, or smaller, it may go through. A
102
- * limit the store names says which, what the call would have taken of it,
103
- * and the bound: `decoded samples`, 120000, 100000.
101
+ * A bound: memory, size or count. Later, or smaller, it may go through. The
102
+ * error names the bound, what the call would have taken of it, and the
103
+ * bound: `set_many documents`, 1001, 1000. `limits` holds the bounds a call
104
+ * meets by those names.
104
105
  */
105
106
  export class LimitError extends TinystoreError {
106
107
  readonly limit: string | undefined
@@ -115,30 +116,91 @@ export class LimitError extends TinystoreError {
115
116
  }
116
117
  }
117
118
 
119
+ /** A bound one call meets: its name, as a LimitError's `limit` says it, and the most it takes. */
120
+ export interface Limit {
121
+ readonly name: string
122
+ readonly bound: number
123
+ }
124
+
125
+ const limit = (name: string, bound: number): Limit => Object.freeze({ name, bound })
126
+
118
127
  /**
119
- * The names a LimitError's `limit` holds, as Go's constants and Python's
120
- * `tinystore.limits` spell them, to map a limit without matching its text.
128
+ * The bounds one call meets, the same in every SDK and in Rust's
129
+ * `tinystore::limits`. A program cuts its batches to them before it calls,
130
+ * and tells which bound a LimitError met without reading its text:
121
131
  *
122
132
  * ```ts
123
- * if (err instanceof LimitError && err.limit === limits.decodedSamples) { … }
133
+ * for (let at = 0; at < courses.length; at += limits.setManyDocuments.bound) { … }
134
+ * if (err instanceof LimitError && err.limit === limits.requestBytes.name) { … }
124
135
  * ```
136
+ *
137
+ * A bound the store's options set, its memory (`store memory`) or its
138
+ * `maxSeries` (`series in a store`), is named the same way and is not here.
125
139
  */
126
140
  export const limits = Object.freeze({
127
- storeMemory: 'store memory',
128
- storeMemoryNow: 'store memory, now',
129
- matchedSeries: 'matched series',
130
- decodedBlocks: 'decoded blocks',
131
- fetchedBytes: 'fetched bytes',
132
- decodedSamples: 'decoded samples',
133
- outputSamples: 'output samples',
134
- outputBuckets: 'output buckets',
135
- recordBytes: "bytes of a record, a block's",
136
- appendBytes: "bytes of records in one Append, a segment's; split it",
137
- jobValueBytes: "bytes of a job's value",
138
- stepBytes: "bytes of a step's answer",
139
- objectBytes: "bytes of an object, the bucket's MaxSize",
140
- rowBytes: "bytes of a call's rows",
141
- } as const)
141
+ /** A bucket's value, as it is stored. */
142
+ valueBytes: limit('value bytes', 1 << 20),
143
+ /** A live setting's change, as JSON. */
144
+ settingBytes: limit('setting bytes', 16 << 10),
145
+ /** Every change a config keeps, together. */
146
+ configBytes: limit('config bytes', 256 << 10),
147
+ /** The callers that wait for one lock or semaphore at once. */
148
+ lockWaiters: limit('lock waiters', 10_000),
149
+ /** The subscriptions to the store's channels at once. */
150
+ subscriptions: limit('subscriptions', 10_000),
151
+ /** A job's payload, its result or a step's answer, as JSON. */
152
+ jobBytes: limit('job bytes', 1 << 20),
153
+ /** A run's progress, as JSON. */
154
+ progressBytes: limit('progress bytes', 4 << 10),
155
+ /** The values one call reads from a database: past them, read a page at a time. */
156
+ rowsBytes: limit("bytes of a call's rows", 64 << 20),
157
+ /** One event, its name, fields and text together. */
158
+ eventBytes: limit('event bytes', 256 << 10),
159
+ /** The events of one append together. */
160
+ appendBytes: limit('append bytes', 4 << 20),
161
+ /** The rows one `aggregate` answers. */
162
+ aggregateRows: limit('aggregate rows', 10_000),
163
+ /** The different values one `top` or `distinct` holds, as JSON. */
164
+ countBytes: limit('count bytes', 64 << 20),
165
+ /** The samples of one `ingest`. */
166
+ ingestSamples: limit('ingest samples', 10_000),
167
+ /** The bytes of one `ingest`'s samples, each with its labels. */
168
+ ingestBytes: limit('ingest bytes', 4 << 20),
169
+ /** The samples of one exporter's text. */
170
+ textSamples: limit('text samples', 100_000),
171
+ /** One exporter's text, and the bytes of its samples. */
172
+ textBytes: limit('text bytes', 32 << 20),
173
+ /** The series one query finds. */
174
+ querySeries: limit('series', 1000),
175
+ /** What one query reads from the file. */
176
+ queryBytes: limit('bytes read', 16 << 20),
177
+ /** The samples one query decodes. */
178
+ querySamples: limit('samples decoded', 1 << 20),
179
+ /** The samples or buckets one query answers. */
180
+ queryAnswer: limit('answer', 100_000),
181
+ /** The documents of one `setMany`. */
182
+ setManyDocuments: limit('set_many documents', 1000),
183
+ /** The documents of one `setMany` as JSON; one request body, `requestBytes`, bounds it first. */
184
+ setManyBytes: limit('set_many bytes', 16 << 20),
185
+ /** One document as JSON, or a table's row as its index reads it. */
186
+ documentBytes: limit('document bytes', 1 << 20),
187
+ /** The hits a search reaches, its offset and its limit together. */
188
+ searchHits: limit('search hits', 1000),
189
+ /** A search's text, in bytes. */
190
+ searchText: limit('search text', 1 << 10),
191
+ /** The conditions of one search's filter. */
192
+ filterConditions: limit('filter conditions', 8),
193
+ /** The values the lists of one search's conditions hold together. */
194
+ filterValues: limit('filter values', 100),
195
+ /** A text a filter keeps or asks for, in bytes. */
196
+ filterText: limit('filter text', 256),
197
+ /** A vector's dimensions. */
198
+ vectorDimensions: limit('vector dimensions', 4096),
199
+ /** One request, its message as the SDK sends it: a larger one fails before it is sent. */
200
+ requestBytes: limit('request bytes', 2 << 20),
201
+ /** One answer, its message as the store sends it. */
202
+ answerBytes: limit('answer bytes', 2 << 20),
203
+ })
142
204
 
143
205
  /** What the call needs is not there: a file to copy or rename. */
144
206
  export class NotFoundError extends TinystoreError {
package/src/index.ts CHANGED
@@ -179,6 +179,8 @@ export {
179
179
  type IndexInfo,
180
180
  type IndexOptions,
181
181
  type IndexStatus,
182
+ type KeyboardLayout,
183
+ type Match as SearchMatch,
182
184
  type Query as SearchQuery,
183
185
  SearchIndex,
184
186
  search,
package/src/kv.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  // its first call on a connection.
5
5
 
6
6
  import type { Connection, Idempotence, Link } from './connection.ts'
7
- import { CorruptError, InvalidError } from './errors.ts'
7
+ import { ConflictError, CorruptError, InvalidError } from './errors.ts'
8
8
  import {
9
9
  checkName,
10
10
  type Home,
@@ -46,6 +46,9 @@ import {
46
46
  */
47
47
  export type ValueType = 'string' | 'bytes' | 'int' | 'bigint' | 'float' | 'bool'
48
48
 
49
+ /** The tries an update makes while other writes come first, as many as a config's change makes. */
50
+ const updateAttempts = 8
51
+
49
52
  export interface BucketOptions {
50
53
  /** a key expires this long after it was written, unless its write gives its own */
51
54
  ttl?: Duration | undefined
@@ -196,6 +199,40 @@ export class Bucket<T> {
196
199
  return this.#ask('kv.create', { key, value: raw, ...writeFields(options) }, 'write', written)
197
200
  }
198
201
 
202
+ /**
203
+ * Writes what `change` makes of the key's value without losing a write
204
+ * made meanwhile: it reads the key, calls `change` with its value, or
205
+ * undefined where it holds none, and writes what `change` returns at the
206
+ * version it read, or creates the key. When another write came first it
207
+ * reads again and calls `change` again, eight times at most, then throws
208
+ * ConflictError: `change` may run more than once, so it must do nothing
209
+ * else. A change that returns undefined, or the very value it was given,
210
+ * writes nothing. It returns the value written, or the one there when
211
+ * nothing was. A key that is there keeps its expiry, and one it creates
212
+ * takes the bucket's, as set and create do.
213
+ *
214
+ * await pinned.update(studentId, courses => [...(courses ?? []), 2518])
215
+ */
216
+ async update(
217
+ key: Key,
218
+ change: (current: T | undefined) => T | undefined,
219
+ ): Promise<T | undefined> {
220
+ for (let attempt = 0; attempt < updateAttempts; attempt++) {
221
+ const entry = await this.entry(key)
222
+ const next = change(entry?.value)
223
+ if (next === undefined || (entry !== undefined && next === entry.value)) {
224
+ return entry?.value
225
+ }
226
+ if (await this.#written(key, next, entry?.version)) {
227
+ return next
228
+ }
229
+ }
230
+ const shown = [this.#home?.describe, `kv bucket ${this.name}`, this.#shownKey(key)]
231
+ throw new ConflictError(
232
+ `${shown.filter(part => part !== undefined).join(': ')}: other writes came first each time it was tried`,
233
+ )
234
+ }
235
+
199
236
  /** Puts the key in a bucket used as a set; false when it was there. */
200
237
  async add(key: Key, options?: Omit<WriteOptions, 'ifVersion'>): Promise<boolean> {
201
238
  return this.#ask('kv.create', { key, ...writeFields(options) }, 'write', written)
@@ -292,6 +329,27 @@ export class Bucket<T> {
292
329
  }
293
330
  }
294
331
 
332
+ // one try of an update: false when another write came first, at the key or creating it
333
+ async #written(key: Key, next: T, version: string | undefined): Promise<boolean> {
334
+ try {
335
+ if (version === undefined) {
336
+ return await this.create(key, next)
337
+ }
338
+ await this.set(key, next, { ifVersion: version })
339
+ return true
340
+ } catch (err) {
341
+ if (err instanceof ConflictError) {
342
+ return false
343
+ }
344
+ throw err
345
+ }
346
+ }
347
+
348
+ // the key as the core's errors show it, after its branch's owners
349
+ #shownKey(key: Key): string {
350
+ return `key ${JSON.stringify([...this.#under, keyText(key)].join('/'))}`
351
+ }
352
+
295
353
  // a call on one key: its fields written where they leave from, its answer read where it arrived
296
354
  #ask<A>(
297
355
  method: Method,
package/src/search.ts CHANGED
@@ -38,6 +38,12 @@ export interface IndexOptions {
38
38
  filters?: readonly string[] | undefined
39
39
  /** a misspelt word read as the index's nearest; on unless false */
40
40
  typoTolerance?: boolean | undefined
41
+ /**
42
+ * the keyboard layouts the index's readers type with, `['ru', 'en']`: a
43
+ * word that finds nothing as typed is read as the same keys type in the
44
+ * other layout, when it finds something there, and `corrected` says so
45
+ */
46
+ layouts?: readonly KeyboardLayout[] | undefined
41
47
  /** a vector the program's model gives each document, and `recall` for searches from clusters */
42
48
  vectors?: { dimensions: number; recall?: number | undefined } | undefined
43
49
  /** how a search by words and a vector at once combines them unless it says */
@@ -46,6 +52,9 @@ export interface IndexOptions {
46
52
  rebuild?: 'automatic' | 'manual' | undefined
47
53
  }
48
54
 
55
+ /** A keyboard layout an index's readers type with: ЙЦУКЕН, or QWERTY. */
56
+ export type KeyboardLayout = 'ru' | 'en'
57
+
49
58
  /** A vector the program's own model gave, of the index's dimensions. */
50
59
  export type Vector = readonly number[] | Float32Array
51
60
 
@@ -88,19 +97,46 @@ export interface Query {
88
97
  /**
89
98
  * the last word read as the start of a word too, as a search box types:
90
99
  * `crypt` finds `cryptography` as well, at half its field's weight; a
91
- * word a space follows is whole, and so is one of a single letter
100
+ * word a space follows is whole, and so is one of a single letter. With
101
+ * `'every'`, every word of two letters or more is read so, the first
102
+ * eight, as people shorten each word of a name: `лин алг` finds
103
+ * `Линейная алгебра`; a document that holds a word for each word of the
104
+ * text comes first, its score raised above the rest
105
+ */
106
+ prefix?: boolean | 'every' | undefined
107
+ /**
108
+ * each hit says where the search's words matched it, in `matches`, as a
109
+ * search box shows them in bold: a word found as written, by its stem or
110
+ * once corrected whole, and a word a prefix began by the letters typed
92
111
  */
93
- prefix?: boolean | undefined
112
+ matches?: boolean | undefined
94
113
  /** 10 unless said; offset and limit together 1000 at most */
95
114
  limit?: number | undefined
96
115
  offset?: number | undefined
97
116
  }
98
117
 
118
+ /**
119
+ * Where a search's word matched a hit: `start` to `end` of a field's text in
120
+ * UTF-16 units, so that `text.slice(start, end)` is what matched.
121
+ */
122
+ export interface Match {
123
+ /** the field's path, or a table's column; `''` for a document that is its own text */
124
+ field: string
125
+ /** the string's place in a field that holds a list */
126
+ item?: number
127
+ start: number
128
+ end: number
129
+ /** the query's word, as its text wrote it */
130
+ word: string
131
+ }
132
+
99
133
  /** A document a search found, and the score that ordered it among this search's hits. */
100
134
  export interface Hit<T> {
101
135
  id: string
102
136
  value: T
103
137
  score: number
138
+ /** where the search's words matched it, when the search asked with `matches` */
139
+ matches?: Match[]
104
140
  }
105
141
 
106
142
  /** What a search found, the best first, and the text it read in fact when a word was misspelt. */
@@ -139,6 +175,7 @@ export interface TableIndexOptions {
139
175
  /** the columns a search may keep its hits to; a boolean is the 1 or 0 SQLite keeps */
140
176
  filters?: readonly string[] | undefined
141
177
  typoTolerance?: boolean | undefined
178
+ layouts?: readonly KeyboardLayout[] | undefined
142
179
  /** a column of blobs of little-endian floats, one a row or null, and `recall` for searches from clusters */
143
180
  vectors?: { column: string; dimensions: number; recall?: number | undefined } | undefined
144
181
  hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
@@ -193,20 +230,29 @@ abstract class Searchable<T> {
193
230
  vector,
194
231
  vectorWeight: asked.hybrid?.vectorWeight,
195
232
  rerank: asked.hybrid?.rerank,
196
- prefix: asked.prefix,
233
+ prefix: asked.prefix === true || undefined,
234
+ // none unless asked, so that a server before it reads the search
235
+ prefixEvery: asked.prefix === 'every' || undefined,
236
+ // JavaScript slices a string by its UTF-16 units
237
+ matches: asked.matches === true ? 'utf16' : undefined,
197
238
  where,
198
239
  offset: asked.offset,
199
240
  limit: asked.limit,
200
241
  })
201
242
  const found = SearchFound.decode(await this.call('search.search', body, 'read'))
202
243
  const columns = found.columns
203
- const hits = (found.hits ?? []).map(hit => ({
204
- id: hit.id ?? '',
205
- value: (columns === undefined
206
- ? parseJson(hit.value ?? 'null')
207
- : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]) as T,
208
- score: hit.score ?? 0,
209
- }))
244
+ const hits = (found.hits ?? []).map(hit => {
245
+ const value = (
246
+ columns === undefined
247
+ ? parseJson(hit.value ?? 'null')
248
+ : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]
249
+ ) as T
250
+ const read: Hit<T> = { id: hit.id ?? '', value, score: hit.score ?? 0 }
251
+ if (hit.matches !== undefined) {
252
+ read.matches = hit.matches.map(matchOf)
253
+ }
254
+ return read
255
+ })
210
256
  return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected }
211
257
  }
212
258
 
@@ -261,7 +307,12 @@ export class SearchIndex<T> extends Searchable<T> {
261
307
  )
262
308
  }
263
309
 
264
- /** Sets every document in one commit, all of them or none: 1,000 and 16 MiB at most. */
310
+ /**
311
+ * Sets every document in one commit, all of them or none: 1,000 at most,
312
+ * `limits.setManyDocuments`, and no more than one request body holds,
313
+ * 2 MiB, `limits.requestBytes`. Past either it fails with a LimitError
314
+ * that names the bound.
315
+ */
265
316
  async setMany(documents: readonly IndexDocument<T>[]): Promise<void> {
266
317
  const encoded = documents.map(document => ({
267
318
  id: keyText(document.id),
@@ -318,6 +369,7 @@ export function tableIndexOpen(
318
369
  fields: fieldsOf(options.fields),
319
370
  filters: [...(options.filters ?? [])],
320
371
  typoTolerance: options.typoTolerance,
372
+ layouts: layoutsOf(options.layouts),
321
373
  dimensions: options.vectors?.dimensions,
322
374
  recall: options.vectors?.recall,
323
375
  vectorWeight: options.hybrid?.vectorWeight,
@@ -337,6 +389,11 @@ function fieldsOf(fields: IndexOptions['fields']): { path: string; weight: numbe
337
389
  : Object.entries(fields ?? {}).map(([path, weight]) => ({ path, weight }))
338
390
  }
339
391
 
392
+ /** The layouts asked for; none left out, so that a server before them reads the open. */
393
+ function layoutsOf(layouts: readonly KeyboardLayout[] | undefined): KeyboardLayout[] | undefined {
394
+ return layouts?.length ? [...layouts] : undefined
395
+ }
396
+
340
397
  /** The open of an index: its name, settings, and whether it reads the index again. */
341
398
  export function indexOpen(name: string, options: IndexOptions = {}, reindex = false): Uint8Array {
342
399
  checkName(name, 'index')
@@ -345,6 +402,7 @@ export function indexOpen(name: string, options: IndexOptions = {}, reindex = fa
345
402
  fields: fieldsOf(options.fields),
346
403
  filters: [...(options.filters ?? [])],
347
404
  typoTolerance: options.typoTolerance,
405
+ layouts: layoutsOf(options.layouts),
348
406
  dimensions: options.vectors?.dimensions,
349
407
  recall: options.vectors?.recall,
350
408
  vectorWeight: options.hybrid?.vectorWeight,
@@ -397,6 +455,25 @@ function statusOf(status: {
397
455
  }
398
456
  }
399
457
 
458
+ function matchOf(found: {
459
+ field?: string | undefined
460
+ item?: number | undefined
461
+ start?: number | undefined
462
+ end?: number | undefined
463
+ word?: string | undefined
464
+ }): Match {
465
+ const match: Match = {
466
+ field: found.field ?? '',
467
+ start: found.start ?? 0,
468
+ end: found.end ?? 0,
469
+ word: found.word ?? '',
470
+ }
471
+ if (found.item !== undefined) {
472
+ match.item = found.item
473
+ }
474
+ return match
475
+ }
476
+
400
477
  function documentJson(name: string, value: unknown): string {
401
478
  const json = JSON.stringify(value)
402
479
  if (json === undefined) {
package/src/session.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  ClosedError,
13
13
  errorOf,
14
14
  LimitError,
15
+ limits,
15
16
  ProtocolError,
16
17
  type TinystoreError,
17
18
  UnavailableError,
@@ -31,10 +32,20 @@ import {
31
32
  putHeader,
32
33
  } from './wire/frame.ts'
33
34
  import { Reader, Writer } from './wire/msgpack.ts'
34
- import { Failure, GoAway, Hello, Welcome } from './wire/protocol.ts'
35
+ import { Failure, GoAway, Hello, methods, Welcome } from './wire/protocol.ts'
35
36
 
36
37
  export const protocol = 1
37
38
 
39
+ /** Each method's name by its number, as the schema spells it: `search.setMany`. */
40
+ const methodNames = new Map<number, string>(
41
+ Object.entries(methods).map(([name, number]) => [number, name]),
42
+ )
43
+
44
+ /** A method's name as the schema spells it, for an error the SDK says itself. */
45
+ function methodName(method: number): string {
46
+ return methodNames.get(method) ?? `method ${method}`
47
+ }
48
+
38
49
  /** What this client lets the server send on a stream before it grants more: a body at least. */
39
50
  export const downloadWindow = 2 << 20
40
51
 
@@ -365,7 +376,12 @@ export class Session {
365
376
  const agreed = await unlessCancelled(this.welcomed, signal)
366
377
  if (body.length > agreed.maxBody) {
367
378
  throw new LimitError(
368
- `a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
379
+ `${methodName(method)}: a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
380
+ {
381
+ limit: limits.requestBytes.name,
382
+ wanted: String(body.length),
383
+ bound: String(agreed.maxBody),
384
+ },
369
385
  )
370
386
  }
371
387
  while (this.#streams.size >= agreed.inFlight && this.#usable() === undefined) {