@tinyshed/tinystore 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/errors.ts CHANGED
@@ -98,9 +98,10 @@ export class InvalidError extends TinystoreError {
98
98
  }
99
99
 
100
100
  /**
101
- * A bound: memory, size or count. Later, or smaller, it may go through. A
102
- * limit the store names says which, what the call would have taken of it,
103
- * and the bound: `decoded samples`, 120000, 100000.
101
+ * A bound: memory, size or count. Later, or smaller, it may go through. The
102
+ * error names the bound, what the call would have taken of it, and the
103
+ * bound: `set_many documents`, 1001, 1000. `limits` holds the bounds a call
104
+ * meets by those names.
104
105
  */
105
106
  export class LimitError extends TinystoreError {
106
107
  readonly limit: string | undefined
@@ -115,30 +116,91 @@ export class LimitError extends TinystoreError {
115
116
  }
116
117
  }
117
118
 
119
+ /** A bound one call meets: its name, as a LimitError's `limit` says it, and the most it takes. */
120
+ export interface Limit {
121
+ readonly name: string
122
+ readonly bound: number
123
+ }
124
+
125
+ const limit = (name: string, bound: number): Limit => Object.freeze({ name, bound })
126
+
118
127
  /**
119
- * The names a LimitError's `limit` holds, as Go's constants and Python's
120
- * `tinystore.limits` spell them, to map a limit without matching its text.
128
+ * The bounds one call meets, the same in every SDK and in Rust's
129
+ * `tinystore::limits`. A program cuts its batches to them before it calls,
130
+ * and tells which bound a LimitError met without reading its text:
121
131
  *
122
132
  * ```ts
123
- * if (err instanceof LimitError && err.limit === limits.decodedSamples) { … }
133
+ * for (let at = 0; at < courses.length; at += limits.setManyDocuments.bound) { … }
134
+ * if (err instanceof LimitError && err.limit === limits.requestBytes.name) { … }
124
135
  * ```
136
+ *
137
+ * A bound the store's options set, its memory (`store memory`) or its
138
+ * `maxSeries` (`series in a store`), is named the same way and is not here.
125
139
  */
126
140
  export const limits = Object.freeze({
127
- storeMemory: 'store memory',
128
- storeMemoryNow: 'store memory, now',
129
- matchedSeries: 'matched series',
130
- decodedBlocks: 'decoded blocks',
131
- fetchedBytes: 'fetched bytes',
132
- decodedSamples: 'decoded samples',
133
- outputSamples: 'output samples',
134
- outputBuckets: 'output buckets',
135
- recordBytes: "bytes of a record, a block's",
136
- appendBytes: "bytes of records in one Append, a segment's; split it",
137
- jobValueBytes: "bytes of a job's value",
138
- stepBytes: "bytes of a step's answer",
139
- objectBytes: "bytes of an object, the bucket's MaxSize",
140
- rowBytes: "bytes of a call's rows",
141
- } as const)
141
+ /** A bucket's value, as it is stored. */
142
+ valueBytes: limit('value bytes', 1 << 20),
143
+ /** A live setting's change, as JSON. */
144
+ settingBytes: limit('setting bytes', 16 << 10),
145
+ /** Every change a config keeps, together. */
146
+ configBytes: limit('config bytes', 256 << 10),
147
+ /** The callers that wait for one lock or semaphore at once. */
148
+ lockWaiters: limit('lock waiters', 10_000),
149
+ /** The subscriptions to the store's channels at once. */
150
+ subscriptions: limit('subscriptions', 10_000),
151
+ /** A job's payload, its result or a step's answer, as JSON. */
152
+ jobBytes: limit('job bytes', 1 << 20),
153
+ /** A run's progress, as JSON. */
154
+ progressBytes: limit('progress bytes', 4 << 10),
155
+ /** The values one call reads from a database: past them, read a page at a time. */
156
+ rowsBytes: limit("bytes of a call's rows", 64 << 20),
157
+ /** One event, its name, fields and text together. */
158
+ eventBytes: limit('event bytes', 256 << 10),
159
+ /** The events of one append together. */
160
+ appendBytes: limit('append bytes', 4 << 20),
161
+ /** The rows one `aggregate` answers. */
162
+ aggregateRows: limit('aggregate rows', 10_000),
163
+ /** The different values one `top` or `distinct` holds, as JSON. */
164
+ countBytes: limit('count bytes', 64 << 20),
165
+ /** The samples of one `ingest`. */
166
+ ingestSamples: limit('ingest samples', 10_000),
167
+ /** The bytes of one `ingest`'s samples, each with its labels. */
168
+ ingestBytes: limit('ingest bytes', 4 << 20),
169
+ /** The samples of one exporter's text. */
170
+ textSamples: limit('text samples', 100_000),
171
+ /** One exporter's text, and the bytes of its samples. */
172
+ textBytes: limit('text bytes', 32 << 20),
173
+ /** The series one query finds. */
174
+ querySeries: limit('series', 1000),
175
+ /** What one query reads from the file. */
176
+ queryBytes: limit('bytes read', 16 << 20),
177
+ /** The samples one query decodes. */
178
+ querySamples: limit('samples decoded', 1 << 20),
179
+ /** The samples or buckets one query answers. */
180
+ queryAnswer: limit('answer', 100_000),
181
+ /** The documents of one `setMany`. */
182
+ setManyDocuments: limit('set_many documents', 1000),
183
+ /** The documents of one `setMany` as JSON; one request body, `requestBytes`, bounds it first. */
184
+ setManyBytes: limit('set_many bytes', 16 << 20),
185
+ /** One document as JSON, or a table's row as its index reads it. */
186
+ documentBytes: limit('document bytes', 1 << 20),
187
+ /** The hits a search reaches, its offset and its limit together. */
188
+ searchHits: limit('search hits', 1000),
189
+ /** A search's text, in bytes. */
190
+ searchText: limit('search text', 1 << 10),
191
+ /** The conditions of one search's filter. */
192
+ filterConditions: limit('filter conditions', 8),
193
+ /** The values the lists of one search's conditions hold together. */
194
+ filterValues: limit('filter values', 100),
195
+ /** A text a filter keeps or asks for, in bytes. */
196
+ filterText: limit('filter text', 256),
197
+ /** A vector's dimensions. */
198
+ vectorDimensions: limit('vector dimensions', 4096),
199
+ /** One request, its message as the SDK sends it: a larger one fails before it is sent. */
200
+ requestBytes: limit('request bytes', 2 << 20),
201
+ /** One answer, its message as the store sends it. */
202
+ answerBytes: limit('answer bytes', 2 << 20),
203
+ })
142
204
 
143
205
  /** What the call needs is not there: a file to copy or rename. */
144
206
  export class NotFoundError extends TinystoreError {
package/src/index.ts CHANGED
@@ -179,6 +179,8 @@ export {
179
179
  type IndexInfo,
180
180
  type IndexOptions,
181
181
  type IndexStatus,
182
+ type KeyboardLayout,
183
+ type Match as SearchMatch,
182
184
  type Query as SearchQuery,
183
185
  SearchIndex,
184
186
  search,
package/src/kv.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  // its first call on a connection.
5
5
 
6
6
  import type { Connection, Idempotence, Link } from './connection.ts'
7
- import { CorruptError, InvalidError } from './errors.ts'
7
+ import { ConflictError, CorruptError, InvalidError } from './errors.ts'
8
8
  import {
9
9
  checkName,
10
10
  type Home,
@@ -46,6 +46,9 @@ import {
46
46
  */
47
47
  export type ValueType = 'string' | 'bytes' | 'int' | 'bigint' | 'float' | 'bool'
48
48
 
49
+ /** The tries an update makes while other writes come first, as many as a config's change makes. */
50
+ const updateAttempts = 8
51
+
49
52
  export interface BucketOptions {
50
53
  /** a key expires this long after it was written, unless its write gives its own */
51
54
  ttl?: Duration | undefined
@@ -196,6 +199,40 @@ export class Bucket<T> {
196
199
  return this.#ask('kv.create', { key, value: raw, ...writeFields(options) }, 'write', written)
197
200
  }
198
201
 
202
+ /**
203
+ * Writes what `change` makes of the key's value without losing a write
204
+ * made meanwhile: it reads the key, calls `change` with its value, or
205
+ * undefined where it holds none, and writes what `change` returns at the
206
+ * version it read, or creates the key. When another write came first it
207
+ * reads again and calls `change` again, eight times at most, then throws
208
+ * ConflictError: `change` may run more than once, so it must do nothing
209
+ * else. A change that returns undefined, or the very value it was given,
210
+ * writes nothing. It returns the value written, or the one there when
211
+ * nothing was. A key that is there keeps its expiry, and one it creates
212
+ * takes the bucket's, as set and create do.
213
+ *
214
+ * await pinned.update(studentId, courses => [...(courses ?? []), 2518])
215
+ */
216
+ async update(
217
+ key: Key,
218
+ change: (current: T | undefined) => T | undefined,
219
+ ): Promise<T | undefined> {
220
+ for (let attempt = 0; attempt < updateAttempts; attempt++) {
221
+ const entry = await this.entry(key)
222
+ const next = change(entry?.value)
223
+ if (next === undefined || (entry !== undefined && next === entry.value)) {
224
+ return entry?.value
225
+ }
226
+ if (await this.#written(key, next, entry?.version)) {
227
+ return next
228
+ }
229
+ }
230
+ const shown = [this.#home?.describe, `kv bucket ${this.name}`, this.#shownKey(key)]
231
+ throw new ConflictError(
232
+ `${shown.filter(part => part !== undefined).join(': ')}: other writes came first each time it was tried`,
233
+ )
234
+ }
235
+
199
236
  /** Puts the key in a bucket used as a set; false when it was there. */
200
237
  async add(key: Key, options?: Omit<WriteOptions, 'ifVersion'>): Promise<boolean> {
201
238
  return this.#ask('kv.create', { key, ...writeFields(options) }, 'write', written)
@@ -292,6 +329,27 @@ export class Bucket<T> {
292
329
  }
293
330
  }
294
331
 
332
+ // one try of an update: false when another write came first, at the key or creating it
333
+ async #written(key: Key, next: T, version: string | undefined): Promise<boolean> {
334
+ try {
335
+ if (version === undefined) {
336
+ return await this.create(key, next)
337
+ }
338
+ await this.set(key, next, { ifVersion: version })
339
+ return true
340
+ } catch (err) {
341
+ if (err instanceof ConflictError) {
342
+ return false
343
+ }
344
+ throw err
345
+ }
346
+ }
347
+
348
+ // the key as the core's errors show it, after its branch's owners
349
+ #shownKey(key: Key): string {
350
+ return `key ${JSON.stringify([...this.#under, keyText(key)].join('/'))}`
351
+ }
352
+
295
353
  // a call on one key: its fields written where they leave from, its answer read where it arrived
296
354
  #ask<A>(
297
355
  method: Method,
package/src/logger.ts CHANGED
@@ -460,12 +460,15 @@ export class Logger {
460
460
  this.#emit({ event: name }, attrs)
461
461
  }
462
462
 
463
- /** The logger of the same stream, buffer and console, its context these fields besides its own. */
463
+ /**
464
+ * The logger of the same stream, buffer and console, its context these
465
+ * fields besides its own: a key it had already takes the new value where
466
+ * it stood, as an object spread does, where it prints, in what is kept
467
+ * and for a query's `where`.
468
+ */
464
469
  with(context: Fields): Logger {
465
- return new Logger(this.#shared, [
466
- ...this.#context,
467
- ...encodeFields(context, this.#shared.redact, this.#shared.replace),
468
- ])
470
+ const added = encodeFields(context, this.#shared.redact, this.#shared.replace)
471
+ return new Logger(this.#shared, onceEach(this.#context, added))
469
472
  }
470
473
 
471
474
  /** Hands what the buffer holds to the server now, as the timer does every second. */
@@ -543,6 +546,27 @@ export class Logger {
543
546
  }
544
547
  }
545
548
 
549
+ /**
550
+ * A context's key, JSON pairs with `added`'s after them, each key once: where
551
+ * it was first given, holding the value given last.
552
+ *
553
+ * ```text
554
+ * component=http user=7, then component=access → component=access user=7
555
+ * ```
556
+ */
557
+ function onceEach(context: readonly string[], added: readonly string[]): string[] {
558
+ const once = [...context]
559
+ for (let i = 0; i < added.length; i += 2) {
560
+ const at = once.findIndex((key, place) => place % 2 === 0 && key === added[i])
561
+ if (at === -1) {
562
+ once.push(added[i] as string, added[i + 1] as string)
563
+ } else {
564
+ once[at + 1] = added[i + 1] as string
565
+ }
566
+ }
567
+ return once
568
+ }
569
+
546
570
  /** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
547
571
  function prints(shared: Shared, level: number): boolean {
548
572
  return shared.print !== undefined && level >= (shared.printed ?? shared.kept)
package/src/search.ts CHANGED
@@ -38,6 +38,12 @@ export interface IndexOptions {
38
38
  filters?: readonly string[] | undefined
39
39
  /** a misspelt word read as the index's nearest; on unless false */
40
40
  typoTolerance?: boolean | undefined
41
+ /**
42
+ * the keyboard layouts the index's readers type with, `['ru', 'en']`: a
43
+ * word that finds nothing as typed is read as the same keys type in the
44
+ * other layout, when it finds something there, and `corrected` says so
45
+ */
46
+ layouts?: readonly KeyboardLayout[] | undefined
41
47
  /** a vector the program's model gives each document, and `recall` for searches from clusters */
42
48
  vectors?: { dimensions: number; recall?: number | undefined } | undefined
43
49
  /** how a search by words and a vector at once combines them unless it says */
@@ -46,6 +52,9 @@ export interface IndexOptions {
46
52
  rebuild?: 'automatic' | 'manual' | undefined
47
53
  }
48
54
 
55
+ /** A keyboard layout an index's readers type with: ЙЦУКЕН, or QWERTY. */
56
+ export type KeyboardLayout = 'ru' | 'en'
57
+
49
58
  /** A vector the program's own model gave, of the index's dimensions. */
50
59
  export type Vector = readonly number[] | Float32Array
51
60
 
@@ -85,16 +94,49 @@ export interface Query {
85
94
  where?: Where | undefined
86
95
  /** this search's way of combining its words and its vector, in place of the index's */
87
96
  hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
97
+ /**
98
+ * the last word read as the start of a word too, as a search box types:
99
+ * `crypt` finds `cryptography` as well, at half its field's weight; a
100
+ * word a space follows is whole, and so is one of a single letter. With
101
+ * `'every'`, every word of two letters or more is read so, the first
102
+ * eight, as people shorten each word of a name: `лин алг` finds
103
+ * `Линейная алгебра`; a document that holds a word for each word of the
104
+ * text comes first, its score raised above the rest
105
+ */
106
+ prefix?: boolean | 'every' | undefined
107
+ /**
108
+ * each hit says where the search's words matched it, in `matches`, as a
109
+ * search box shows them in bold: a word found as written, by its stem or
110
+ * once corrected whole, and a word a prefix began by the letters typed
111
+ */
112
+ matches?: boolean | undefined
88
113
  /** 10 unless said; offset and limit together 1000 at most */
89
114
  limit?: number | undefined
90
115
  offset?: number | undefined
91
116
  }
92
117
 
118
+ /**
119
+ * Where a search's word matched a hit: `start` to `end` of a field's text in
120
+ * UTF-16 units, so that `text.slice(start, end)` is what matched.
121
+ */
122
+ export interface Match {
123
+ /** the field's path, or a table's column; `''` for a document that is its own text */
124
+ field: string
125
+ /** the string's place in a field that holds a list */
126
+ item?: number
127
+ start: number
128
+ end: number
129
+ /** the query's word, as its text wrote it */
130
+ word: string
131
+ }
132
+
93
133
  /** A document a search found, and the score that ordered it among this search's hits. */
94
134
  export interface Hit<T> {
95
135
  id: string
96
136
  value: T
97
137
  score: number
138
+ /** where the search's words matched it, when the search asked with `matches` */
139
+ matches?: Match[]
98
140
  }
99
141
 
100
142
  /** What a search found, the best first, and the text it read in fact when a word was misspelt. */
@@ -133,6 +175,7 @@ export interface TableIndexOptions {
133
175
  /** the columns a search may keep its hits to; a boolean is the 1 or 0 SQLite keeps */
134
176
  filters?: readonly string[] | undefined
135
177
  typoTolerance?: boolean | undefined
178
+ layouts?: readonly KeyboardLayout[] | undefined
136
179
  /** a column of blobs of little-endian floats, one a row or null, and `recall` for searches from clusters */
137
180
  vectors?: { column: string; dimensions: number; recall?: number | undefined } | undefined
138
181
  hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
@@ -187,19 +230,29 @@ abstract class Searchable<T> {
187
230
  vector,
188
231
  vectorWeight: asked.hybrid?.vectorWeight,
189
232
  rerank: asked.hybrid?.rerank,
233
+ prefix: asked.prefix === true || undefined,
234
+ // none unless asked, so that a server before it reads the search
235
+ prefixEvery: asked.prefix === 'every' || undefined,
236
+ // JavaScript slices a string by its UTF-16 units
237
+ matches: asked.matches === true ? 'utf16' : undefined,
190
238
  where,
191
239
  offset: asked.offset,
192
240
  limit: asked.limit,
193
241
  })
194
242
  const found = SearchFound.decode(await this.call('search.search', body, 'read'))
195
243
  const columns = found.columns
196
- const hits = (found.hits ?? []).map(hit => ({
197
- id: hit.id ?? '',
198
- value: (columns === undefined
199
- ? parseJson(hit.value ?? 'null')
200
- : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]) as T,
201
- score: hit.score ?? 0,
202
- }))
244
+ const hits = (found.hits ?? []).map(hit => {
245
+ const value = (
246
+ columns === undefined
247
+ ? parseJson(hit.value ?? 'null')
248
+ : rowsOf<T>(columns, [(hit.row ?? []) as SqlValue[]])[0]
249
+ ) as T
250
+ const read: Hit<T> = { id: hit.id ?? '', value, score: hit.score ?? 0 }
251
+ if (hit.matches !== undefined) {
252
+ read.matches = hit.matches.map(matchOf)
253
+ }
254
+ return read
255
+ })
203
256
  return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected }
204
257
  }
205
258
 
@@ -254,7 +307,12 @@ export class SearchIndex<T> extends Searchable<T> {
254
307
  )
255
308
  }
256
309
 
257
- /** Sets every document in one commit, all of them or none: 1,000 and 16 MiB at most. */
310
+ /**
311
+ * Sets every document in one commit, all of them or none: 1,000 at most,
312
+ * `limits.setManyDocuments`, and no more than one request body holds,
313
+ * 2 MiB, `limits.requestBytes`. Past either it fails with a LimitError
314
+ * that names the bound.
315
+ */
258
316
  async setMany(documents: readonly IndexDocument<T>[]): Promise<void> {
259
317
  const encoded = documents.map(document => ({
260
318
  id: keyText(document.id),
@@ -311,6 +369,7 @@ export function tableIndexOpen(
311
369
  fields: fieldsOf(options.fields),
312
370
  filters: [...(options.filters ?? [])],
313
371
  typoTolerance: options.typoTolerance,
372
+ layouts: layoutsOf(options.layouts),
314
373
  dimensions: options.vectors?.dimensions,
315
374
  recall: options.vectors?.recall,
316
375
  vectorWeight: options.hybrid?.vectorWeight,
@@ -330,6 +389,11 @@ function fieldsOf(fields: IndexOptions['fields']): { path: string; weight: numbe
330
389
  : Object.entries(fields ?? {}).map(([path, weight]) => ({ path, weight }))
331
390
  }
332
391
 
392
+ /** The layouts asked for; none left out, so that a server before them reads the open. */
393
+ function layoutsOf(layouts: readonly KeyboardLayout[] | undefined): KeyboardLayout[] | undefined {
394
+ return layouts?.length ? [...layouts] : undefined
395
+ }
396
+
333
397
  /** The open of an index: its name, settings, and whether it reads the index again. */
334
398
  export function indexOpen(name: string, options: IndexOptions = {}, reindex = false): Uint8Array {
335
399
  checkName(name, 'index')
@@ -338,6 +402,7 @@ export function indexOpen(name: string, options: IndexOptions = {}, reindex = fa
338
402
  fields: fieldsOf(options.fields),
339
403
  filters: [...(options.filters ?? [])],
340
404
  typoTolerance: options.typoTolerance,
405
+ layouts: layoutsOf(options.layouts),
341
406
  dimensions: options.vectors?.dimensions,
342
407
  recall: options.vectors?.recall,
343
408
  vectorWeight: options.hybrid?.vectorWeight,
@@ -390,6 +455,25 @@ function statusOf(status: {
390
455
  }
391
456
  }
392
457
 
458
+ function matchOf(found: {
459
+ field?: string | undefined
460
+ item?: number | undefined
461
+ start?: number | undefined
462
+ end?: number | undefined
463
+ word?: string | undefined
464
+ }): Match {
465
+ const match: Match = {
466
+ field: found.field ?? '',
467
+ start: found.start ?? 0,
468
+ end: found.end ?? 0,
469
+ word: found.word ?? '',
470
+ }
471
+ if (found.item !== undefined) {
472
+ match.item = found.item
473
+ }
474
+ return match
475
+ }
476
+
393
477
  function documentJson(name: string, value: unknown): string {
394
478
  const json = JSON.stringify(value)
395
479
  if (json === undefined) {
package/src/session.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  ClosedError,
13
13
  errorOf,
14
14
  LimitError,
15
+ limits,
15
16
  ProtocolError,
16
17
  type TinystoreError,
17
18
  UnavailableError,
@@ -31,10 +32,20 @@ import {
31
32
  putHeader,
32
33
  } from './wire/frame.ts'
33
34
  import { Reader, Writer } from './wire/msgpack.ts'
34
- import { Failure, GoAway, Hello, Welcome } from './wire/protocol.ts'
35
+ import { Failure, GoAway, Hello, methods, Welcome } from './wire/protocol.ts'
35
36
 
36
37
  export const protocol = 1
37
38
 
39
+ /** Each method's name by its number, as the schema spells it: `search.setMany`. */
40
+ const methodNames = new Map<number, string>(
41
+ Object.entries(methods).map(([name, number]) => [number, name]),
42
+ )
43
+
44
+ /** A method's name as the schema spells it, for an error the SDK says itself. */
45
+ function methodName(method: number): string {
46
+ return methodNames.get(method) ?? `method ${method}`
47
+ }
48
+
38
49
  /** What this client lets the server send on a stream before it grants more: a body at least. */
39
50
  export const downloadWindow = 2 << 20
40
51
 
@@ -365,7 +376,12 @@ export class Session {
365
376
  const agreed = await unlessCancelled(this.welcomed, signal)
366
377
  if (body.length > agreed.maxBody) {
367
378
  throw new LimitError(
368
- `a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
379
+ `${methodName(method)}: a request of ${body.length} bytes, past the ${agreed.maxBody} a body holds`,
380
+ {
381
+ limit: limits.requestBytes.name,
382
+ wanted: String(body.length),
383
+ bound: String(agreed.maxBody),
384
+ },
369
385
  )
370
386
  }
371
387
  while (this.#streams.size >= agreed.inFlight && this.#usable() === undefined) {