@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/src/session.ts CHANGED
@@ -23,11 +23,14 @@ import {
23
23
  Flag,
24
24
  frame,
25
25
  granted,
26
+ headerSize,
26
27
  Kind,
27
28
  kindName,
28
29
  parseHeader,
30
+ putHeader,
29
31
  } from './wire/frame.ts'
30
- import { Failure, GoAway, Hello, Welcome } from './wire/messages.ts'
32
+ import { Reader, Writer } from './wire/msgpack.ts'
33
+ import { Failure, GoAway, Hello, Welcome } from './wire/protocol.ts'
31
34
 
32
35
  export const protocol = 1
33
36
 
@@ -37,7 +40,16 @@ export const downloadWindow = 2 << 20
37
40
  /** The largest body a WELCOME may come in, before a body is agreed. */
38
41
  const welcomeMost = 1 << 20
39
42
 
40
- export type Agreed = Required<Omit<Read<typeof Welcome.fields>, 'proof'>>
43
+ /** A frame past this leaves by itself, so that what a turn's calls are written into stays small. */
44
+ const largeFrame = 1 << 15
45
+
46
+ export type Agreed = Given<Omit<Read<typeof Welcome.fields>, 'proof' | 'durability'>> & {
47
+ /** how far the store's files' commits go, when it was opened with one: full or os */
48
+ durability: string | undefined
49
+ }
50
+
51
+ /** Every field of a message read, each with a value: one the message left out is at its zero. */
52
+ type Given<T> = { [K in keyof T]-?: Exclude<T[K], undefined> }
41
53
 
42
54
  export interface SessionOptions {
43
55
  /** a name and version, for the server's logs */
@@ -56,7 +68,7 @@ export interface SessionOptions {
56
68
  * and one that had is a read to send again or a write whose outcome is
57
69
  * unknown.
58
70
  */
59
- export class LostError extends ClosedError {
71
+ export class ConnectionLostError extends ClosedError {
60
72
  readonly sent: boolean
61
73
 
62
74
  constructor(message: string, sent: boolean) {
@@ -212,6 +224,27 @@ export class Stream {
212
224
  #unwatch: (() => void) | undefined
213
225
  }
214
226
 
227
+ /**
228
+ * A call that left the moment it was made, a REQUEST the RESPONSE answers:
229
+ * its answer is read where it arrives and given to whoever asked, with no
230
+ * stream's state between. Nothing cancels it.
231
+ */
232
+ class Direct {
233
+ readonly read: (r: Reader) => unknown
234
+ readonly resolve: (value: never) => void
235
+ readonly reject: (err: Error) => void
236
+
237
+ constructor(
238
+ read: (r: Reader) => unknown,
239
+ resolve: (value: never) => void,
240
+ reject: (err: Error) => void,
241
+ ) {
242
+ this.read = read
243
+ this.resolve = resolve
244
+ this.reject = reject
245
+ }
246
+ }
247
+
215
248
  /** A frame waiting for the connection's credit, in the order it was asked for. */
216
249
  interface Queued {
217
250
  stream: Stream
@@ -233,15 +266,18 @@ export class Session {
233
266
  #welcome: Waiter<Agreed>
234
267
  #agreed: Agreed | undefined
235
268
 
236
- #streams = new Map<number, Stream>()
269
+ #streams = new Map<number, Stream | Direct>()
237
270
  #next = 0
238
271
  #waitingToOpen: Waiter<void>[] = []
239
272
  #queue: Queued[] = []
240
273
  #credit = 0
241
274
 
242
- #out: Uint8Array[] = []
243
- #outBytes = 0
275
+ // what leaves in this turn's one write: every frame asked for is written
276
+ // here, a call's body straight from its fields
277
+ #out = new Writer(1 << 12)
244
278
  #flushing = false
279
+ readonly #leave = () => this.#flush()
280
+ readonly #reader = new Reader(new Uint8Array(0))
245
281
 
246
282
  #rest: Uint8Array | undefined
247
283
  #goingAway = false
@@ -340,7 +376,7 @@ export class Session {
340
376
  /** Why no stream may open now: the connection ended or is going away. */
341
377
  #usable(): Error | undefined {
342
378
  if (this.#ended !== undefined) {
343
- return new LostError(`the connection ended: ${this.#ended.message}`, false)
379
+ return new ConnectionLostError(`the connection ended: ${this.#ended.message}`, false)
344
380
  }
345
381
  if (this.#goingAway) {
346
382
  return new UnavailableError('the server is closing this connection')
@@ -348,6 +384,57 @@ export class Session {
348
384
  return undefined
349
385
  }
350
386
 
387
+ /**
388
+ * Makes a call in place when nothing makes it wait: `write` writes its
389
+ * body into the bytes that leave next, and `read` reads its answer where
390
+ * it arrived, for `resolve`. It says false, having sent nothing, when the
391
+ * call must wait or may be cancelled: before the WELCOME, under a signal,
392
+ * with every stream in use, or past the connection's credit. Then `call`
393
+ * makes it.
394
+ *
395
+ * A call an event loop makes costs it a promise and nothing more this way.
396
+ * Through a stream it cost five, and a copy of its body three times.
397
+ */
398
+ direct<T>(
399
+ method: number,
400
+ write: (w: Writer) => void,
401
+ read: (r: Reader) => T,
402
+ resolve: (value: T) => void,
403
+ reject: (err: Error) => void,
404
+ ): boolean {
405
+ const agreed = this.#agreed
406
+ if (
407
+ agreed === undefined ||
408
+ this.#ended !== undefined ||
409
+ this.#goingAway ||
410
+ this.#queue.length > 0 ||
411
+ this.#streams.size >= agreed.inFlight ||
412
+ currentSignal() !== undefined
413
+ ) {
414
+ return false
415
+ }
416
+ const out = this.#out
417
+ const start = out.reserve(headerSize)
418
+ let length: number
419
+ try {
420
+ write(out)
421
+ length = out.length - start - headerSize
422
+ } catch (err) {
423
+ out.truncate(start)
424
+ throw err
425
+ }
426
+ if (length > agreed.maxBody || length > this.#credit) {
427
+ out.truncate(start)
428
+ return false
429
+ }
430
+ const id = this.#nextID()
431
+ putHeader(out.buffer, start, Kind.request, Flag.end, method, id, length)
432
+ this.#credit -= length
433
+ this.#streams.set(id, new Direct(read, resolve as (value: never) => void, reject))
434
+ this.#soon()
435
+ return true
436
+ }
437
+
351
438
  /** A call: one REQUEST, and the RESPONSE that answers it. */
352
439
  async call(method: number, body: Uint8Array, signal?: AbortSignal): Promise<Uint8Array> {
353
440
  signal?.throwIfAborted()
@@ -408,11 +495,19 @@ export class Session {
408
495
  if (this.#ended !== undefined) {
409
496
  return
410
497
  }
411
- this.#out.push(bytes)
412
- this.#outBytes += bytes.length
498
+ if (bytes.length > largeFrame) {
499
+ this.#flush()
500
+ this.#send(bytes)
501
+ return
502
+ }
503
+ this.#out.raw(bytes)
504
+ this.#soon()
505
+ }
506
+
507
+ #soon(): void {
413
508
  if (!this.#flushing) {
414
509
  this.#flushing = true
415
- queueMicrotask(() => this.#flush())
510
+ queueMicrotask(this.#leave)
416
511
  }
417
512
  }
418
513
 
@@ -422,19 +517,8 @@ export class Session {
422
517
  if (this.#out.length === 0 || this.#ended !== undefined) {
423
518
  return
424
519
  }
425
- let bytes: Uint8Array
426
- if (this.#out.length === 1) {
427
- bytes = this.#out[0]!
428
- } else {
429
- bytes = new Uint8Array(this.#outBytes)
430
- let at = 0
431
- for (const each of this.#out) {
432
- bytes.set(each, at)
433
- at += each.length
434
- }
435
- }
436
- this.#out = []
437
- this.#outBytes = 0
520
+ const bytes = this.#out.bytes()
521
+ this.#out.truncate(0)
438
522
  this.#send(bytes)
439
523
  }
440
524
 
@@ -470,38 +554,53 @@ export class Session {
470
554
  bytes.set(chunk, this.#rest.length)
471
555
  this.#rest = undefined
472
556
  }
557
+ const at = this.read(bytes, bytes.length)
558
+ if (at < bytes.length && this.#ended === undefined) {
559
+ this.#rest = bytes.slice(at)
560
+ }
561
+ }
562
+
563
+ /**
564
+ * Takes the frames whole in the first `length` bytes and says how many
565
+ * bytes they were: the rest is a frame's first part, which whoever owns
566
+ * the bytes keeps for its end. Nothing of the bytes is kept here, so their
567
+ * owner may write them over once this returns.
568
+ */
569
+ read(bytes: Uint8Array, length: number): number {
570
+ if (this.#ended !== undefined) {
571
+ return length
572
+ }
473
573
  let at = 0
474
574
  try {
475
- while (bytes.length - at >= 12) {
575
+ while (length - at >= headerSize) {
476
576
  const h = parseHeader(bytes, at)
477
577
  checkHeader(h, this.#agreed?.maxBody ?? welcomeMost)
478
- const stop = at + 12 + h.length
479
- if (stop > bytes.length) {
578
+ const stop = at + headerSize + h.length
579
+ if (stop > length) {
480
580
  break
481
581
  }
482
- // a body of its own, since events outlive the read's buffer
483
- this.#take(h, bytes.slice(at + 12, stop))
582
+ this.#take(h, bytes, at + headerSize, stop)
484
583
  at = stop
485
584
  }
486
585
  } catch (err) {
487
586
  this.fault(err instanceof Error ? err : new ProtocolError(String(err)))
488
- return
489
- }
490
- if (at < bytes.length) {
491
- this.#rest = bytes.slice(at)
587
+ return length
492
588
  }
589
+ return at
493
590
  }
494
591
 
495
- #take(h: ReturnType<typeof parseHeader>, body: Uint8Array): void {
592
+ // takes the frame whose body is the bytes from `from` to `to`
593
+ #take(h: ReturnType<typeof parseHeader>, bytes: Uint8Array, from: number, to: number): void {
594
+ if (this.#agreed !== undefined && (h.kind === Kind.response || h.kind === Kind.data)) {
595
+ this.#answer(h, bytes, from, to)
596
+ return
597
+ }
598
+ const body = bytes.slice(from, to)
496
599
  if (this.#agreed === undefined) {
497
600
  this.#handshake(h.kind, body)
498
601
  return
499
602
  }
500
603
  switch (h.kind) {
501
- case Kind.response:
502
- case Kind.data:
503
- this.#answer(h.kind, h.flags, h.stream, body)
504
- return
505
604
  case Kind.credit:
506
605
  this.#credited(h.stream, granted(body))
507
606
  return
@@ -553,17 +652,25 @@ export class Session {
553
652
  streamCredit: welcome.streamCredit ?? 0,
554
653
  engines: welcome.engines ?? [],
555
654
  now: welcome.now ?? 0,
655
+ durability: welcome.durability,
556
656
  }
557
657
  this.#agreed = agreed
558
658
  this.#credit = agreed.connectionCredit
559
659
  this.#welcome.resolve(agreed)
560
660
  }
561
661
 
562
- #answer(kind: number, flags: number, id: number, body: Uint8Array): void {
662
+ #answer(h: ReturnType<typeof parseHeader>, bytes: Uint8Array, from: number, to: number): void {
663
+ const { kind, flags, stream: id } = h
563
664
  const stream = this.#streams.get(id)
564
665
  if (stream === undefined) {
565
666
  throw new ProtocolError(`a ${kindName(kind)} on stream ${id}, which is not in use`)
566
667
  }
668
+ if (stream instanceof Direct) {
669
+ this.#answerDirect(stream, h, bytes, from, to)
670
+ return
671
+ }
672
+ // a body of its own, since events outlive the read's buffer
673
+ const body = bytes.slice(from, to)
567
674
  const end = (flags & Flag.end) !== 0
568
675
  if (end) {
569
676
  this.#release(stream)
@@ -578,13 +685,48 @@ export class Session {
578
685
  }
579
686
  }
580
687
 
688
+ // reads a call's answer where it arrived: what `read` returns holds none of these bytes
689
+ #answerDirect(
690
+ call: Direct,
691
+ h: ReturnType<typeof parseHeader>,
692
+ bytes: Uint8Array,
693
+ from: number,
694
+ to: number,
695
+ ): void {
696
+ if (h.kind !== Kind.response || (h.flags & Flag.end) === 0) {
697
+ throw new ProtocolError(
698
+ `a ${kindName(h.kind)} without END answering a call on stream ${h.stream}`,
699
+ )
700
+ }
701
+ this.#streams.delete(h.stream)
702
+ this.#waitingToOpen.shift()?.resolve()
703
+ if ((h.flags & Flag.error) !== 0) {
704
+ call.reject(failureOf(bytes.slice(from, to)))
705
+ return
706
+ }
707
+ const reader = this.#reader
708
+ reader.reset(bytes, from, to)
709
+ let value: unknown
710
+ try {
711
+ value = call.read(reader)
712
+ reader.end()
713
+ } catch (err) {
714
+ call.reject(err instanceof Error ? err : new ProtocolError(String(err)))
715
+ return
716
+ }
717
+ call.resolve(value as never)
718
+ }
719
+
581
720
  #credited(id: number, n: number): void {
582
721
  if (id === 0) {
583
722
  this.#credit += n
584
723
  this.#drain()
585
724
  return
586
725
  }
587
- this.#streams.get(id)?.grant(n)
726
+ const stream = this.#streams.get(id)
727
+ if (stream instanceof Stream) {
728
+ stream.grant(n)
729
+ }
588
730
  }
589
731
 
590
732
  // after a GOAWAY no stream opens; a REQUEST still waiting for credit goes
@@ -618,13 +760,17 @@ export class Session {
618
760
  this.#ended = err
619
761
  this.#welcome.reject(err)
620
762
  for (const stream of this.#streams.values()) {
621
- stream.fail(new LostError(`the connection ended: ${err.message}`, stream.sent))
763
+ if (stream instanceof Direct) {
764
+ stream.reject(new ConnectionLostError(`the connection ended: ${err.message}`, true))
765
+ continue
766
+ }
767
+ stream.fail(new ConnectionLostError(`the connection ended: ${err.message}`, stream.sent))
622
768
  stream.finish()
623
769
  }
624
770
  this.#streams.clear()
625
771
  this.#queue = []
626
772
  for (const w of this.#waitingToOpen.splice(0)) {
627
- w.reject(new LostError(`the connection ended: ${err.message}`, false))
773
+ w.reject(new ConnectionLostError(`the connection ended: ${err.message}`, false))
628
774
  }
629
775
  for (const listener of this.#endListeners.splice(0)) {
630
776
  listener(err)