@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
@@ -0,0 +1,99 @@
1
+ // The core in this process under Node, through its addon, tinystore.node:
2
+ // the frames a socket would carry go to the addon's send, which hands back
3
+ // what is ready at once, and the core's wake tells the loop to read the rest.
4
+ // Node alone: Bun loads the C library through bun:ffi instead.
5
+
6
+ import { createRequire } from 'node:module'
7
+
8
+ import { ClosedError } from '../errors.ts'
9
+ import type { Transport } from '../runtime.ts'
10
+
11
+ /** What tinystore.node exports: crates/ffi built with `node`. */
12
+ interface Addon {
13
+ open(dir: string, options: Uint8Array, wake: () => void): AddonPipe
14
+ }
15
+
16
+ interface AddonPipe {
17
+ send(frames: Uint8Array): Uint8Array | null
18
+ recv(): Uint8Array | null
19
+ close(): void
20
+ }
21
+
22
+ /** What the pipe tells the session reading it. */
23
+ export interface AddonEvents {
24
+ /** bytes the core wrote: any part of a frame, or several */
25
+ data(bytes: Uint8Array): void
26
+ /** the store closed; called once */
27
+ end(err: Error): void
28
+ }
29
+
30
+ /**
31
+ * Opens a pipe to the store in a directory, the core loaded from the addon
32
+ * at `path`. `options` is the wire's store.Options, how this process opens
33
+ * the store.
34
+ */
35
+ export function openAddon(
36
+ path: string,
37
+ dir: string,
38
+ options: Uint8Array,
39
+ events: AddonEvents,
40
+ ): Transport {
41
+ const addon = createRequire(import.meta.url)(path) as Addon
42
+
43
+ // What the session writes while it reads: its frames wait for the read's
44
+ // end, since an answer handed back meanwhile would reach the session in
45
+ // the middle of the bytes it is reading.
46
+ let reading = false
47
+ const waiting: Uint8Array[] = []
48
+
49
+ const take = (bytes: Uint8Array | null): void => {
50
+ reading = true
51
+ try {
52
+ let next = bytes
53
+ while (next !== null && pipe !== undefined) {
54
+ events.data(next)
55
+ next = pipe.recv()
56
+ }
57
+ } finally {
58
+ reading = false
59
+ }
60
+ const queued = waiting.shift()
61
+ if (queued !== undefined) {
62
+ send(queued)
63
+ }
64
+ }
65
+
66
+ const send = (bytes: Uint8Array): void => {
67
+ if (pipe === undefined) {
68
+ return
69
+ }
70
+ if (reading) {
71
+ waiting.push(bytes)
72
+ return
73
+ }
74
+ take(pipe.send(bytes))
75
+ }
76
+
77
+ let pipe: AddonPipe | undefined
78
+ try {
79
+ pipe = addon.open(dir, options, () => {
80
+ if (!reading) {
81
+ take(pipe?.recv() ?? null)
82
+ }
83
+ })
84
+ } catch (err) {
85
+ throw new ClosedError(`cannot open ${dir}: ${(err as Error).message}`)
86
+ }
87
+ return {
88
+ write: send,
89
+ close() {
90
+ const closing = pipe
91
+ if (closing === undefined) {
92
+ return
93
+ }
94
+ pipe = undefined
95
+ closing.close()
96
+ events.end(new ClosedError('the store closed'))
97
+ },
98
+ }
99
+ }
@@ -8,7 +8,7 @@ import net from 'node:net'
8
8
  import manifest from '../../package.json' with { type: 'json' }
9
9
  import { ClosedError } from '../errors.ts'
10
10
  import type { PrivateChild, Runtime, TlsOptions, Transport, TransportEvents } from '../runtime.ts'
11
- import { address, exitedWith, spawnDetached, stderrKept } from './shared.ts'
11
+ import { address, exitedWith, platformPackage, spawnDetached, stderrKept } from './shared.ts'
12
12
 
13
13
  export const bunRuntime: Runtime = {
14
14
  get client() {
@@ -30,7 +30,7 @@ export const bunRuntime: Runtime = {
30
30
  }
31
31
  },
32
32
  which: name => Bun.which(name) ?? undefined,
33
- packagedBinary,
33
+ packaged,
34
34
  }
35
35
 
36
36
  async function connect(
@@ -178,12 +178,10 @@ function spawnPrivate(argv: string[], events: TransportEvents): PrivateChild {
178
178
  }
179
179
  }
180
180
 
181
- /** The platform's package of the server binary, installed beside this one. */
182
- function packagedBinary(): string | undefined {
183
- const name = `@tinyshed/tinystore-${process.platform}-${process.arch}`
184
- const binary = process.platform === 'win32' ? 'tinystore.exe' : 'tinystore'
181
+ /** A file of the platform's package, installed beside this one. */
182
+ function packaged(file: string): string | undefined {
185
183
  try {
186
- return Bun.resolveSync(`${name}/bin/${binary}`, import.meta.dir)
184
+ return Bun.resolveSync(`${platformPackage()}/${file}`, import.meta.dir)
187
185
  } catch {
188
186
  return undefined
189
187
  }
@@ -13,7 +13,7 @@ import tls from 'node:tls'
13
13
  import manifest from '../../package.json' with { type: 'json' }
14
14
  import { ClosedError } from '../errors.ts'
15
15
  import type { PrivateChild, Runtime, TlsOptions, Transport, TransportEvents } from '../runtime.ts'
16
- import { address, exitedWith, spawnDetached, stderrKept } from './shared.ts'
16
+ import { address, exitedWith, platformPackage, spawnDetached, stderrKept } from './shared.ts'
17
17
 
18
18
  export const nodeRuntime: Runtime = {
19
19
  client: `tinystore-js/${manifest.version} node/${process.versions.node}`,
@@ -32,7 +32,7 @@ export const nodeRuntime: Runtime = {
32
32
  }
33
33
  },
34
34
  which,
35
- packagedBinary,
35
+ packaged,
36
36
  }
37
37
 
38
38
  function connect(
@@ -165,12 +165,10 @@ function executable(path: string, windows: boolean): boolean {
165
165
  }
166
166
  }
167
167
 
168
- /** The platform's package of the server binary, installed beside this one. */
169
- function packagedBinary(): string | undefined {
170
- const name = `@tinyshed/tinystore-${process.platform}-${process.arch}`
171
- const binary = process.platform === 'win32' ? 'tinystore.exe' : 'tinystore'
168
+ /** A file of the platform's package, installed beside this one. */
169
+ function packaged(file: string): string | undefined {
172
170
  try {
173
- return createRequire(import.meta.url).resolve(`${name}/bin/${binary}`)
171
+ return createRequire(import.meta.url).resolve(`${platformPackage()}/${file}`)
174
172
  } catch {
175
173
  return undefined
176
174
  }
@@ -0,0 +1,178 @@
1
+ // The core in this process, through its C ABI: the frames a socket would
2
+ // carry go to tinystore_send, and those the core owes are written into bytes
3
+ // of ours, by that call or by tinystore_recv once the core wakes us. No
4
+ // memory of the core's comes back: it reads our frames where they are and
5
+ // writes its own where we say. Bun alone: bun:ffi.
6
+
7
+ import { dlopen, FFIType, JSCallback, type Pointer, ptr } from 'bun:ffi'
8
+
9
+ import { ClosedError } from '../errors.ts'
10
+ import type { Transport } from '../runtime.ts'
11
+
12
+ const symbols = {
13
+ tinystore_open: {
14
+ args: [
15
+ FFIType.ptr,
16
+ FFIType.u64,
17
+ FFIType.ptr,
18
+ FFIType.u64,
19
+ FFIType.function,
20
+ FFIType.ptr,
21
+ FFIType.ptr,
22
+ FFIType.ptr,
23
+ FFIType.u64,
24
+ ],
25
+ returns: FFIType.u64_fast,
26
+ },
27
+ tinystore_send: {
28
+ args: [FFIType.ptr, FFIType.ptr, FFIType.u64, FFIType.ptr, FFIType.u64],
29
+ returns: FFIType.u64_fast,
30
+ },
31
+ tinystore_recv: {
32
+ args: [FFIType.ptr, FFIType.u32, FFIType.ptr, FFIType.u64],
33
+ returns: FFIType.u64_fast,
34
+ },
35
+ tinystore_close: { args: [FFIType.ptr], returns: FFIType.void },
36
+ } as const
37
+
38
+ /** What the pipe tells the session reading it. */
39
+ export interface PipeEvents {
40
+ /**
41
+ * Reads the frames whole in the first `length` bytes and says how many
42
+ * bytes they were. The bytes are the pipe's, written over by its next
43
+ * read: nothing of them may be kept.
44
+ */
45
+ frames(bytes: Uint8Array, length: number): number
46
+ /** the store closed; called once */
47
+ end(err: Error): void
48
+ }
49
+
50
+ /** The room for what the core owes, to begin with and once a large frame has passed. */
51
+ const room = 1 << 16
52
+ /** Room past this is given back once nothing is held in it. */
53
+ const kept = 1 << 20
54
+
55
+ /**
56
+ * Opens a pipe to the store in a directory, the core loaded from `library`.
57
+ * `options` is the wire's store.Options, how this process opens the store.
58
+ */
59
+ export function openPipe(
60
+ library: string,
61
+ dir: string,
62
+ options: Uint8Array,
63
+ events: PipeEvents,
64
+ ): Transport {
65
+ const lib = dlopen(library, symbols)
66
+ const name = new TextEncoder().encode(dir)
67
+ const opened = new BigUint64Array(1)
68
+ const failure = new Uint8Array(1024)
69
+ let connection: Pointer | null = null
70
+
71
+ // where the core writes what it owes: whole frames are read from the
72
+ // front, and a frame's first part is held there for its rest
73
+ let inbox = new Uint8Array(room)
74
+ let held = 0
75
+
76
+ // reads the frames among the `written` bytes behind what is held, and
77
+ // says whether the core filled the room, which means more may wait
78
+ const read = (written: number): boolean => {
79
+ const length = held + written
80
+ const filled = length === inbox.length
81
+ const taken = written === 0 ? 0 : events.frames(inbox, length)
82
+ inbox.copyWithin(0, taken, length)
83
+ held = length - taken
84
+ if (held === inbox.length) {
85
+ // a frame longer than the room: twice the room, until it fits
86
+ const grown = new Uint8Array(inbox.length * 2)
87
+ grown.set(inbox)
88
+ inbox = grown
89
+ } else if (held === 0 && inbox.length > kept) {
90
+ inbox = new Uint8Array(room)
91
+ }
92
+ return filled
93
+ }
94
+
95
+ // What the session writes while it reads: its frames wait for the read's
96
+ // end, since a call of the core made meanwhile would write over the bytes
97
+ // being read.
98
+ let reading = false
99
+ const waiting: Uint8Array[] = []
100
+
101
+ // reads what a call of the core wrote, and on while the core fills the
102
+ // room, since no wake comes for what waits; then sends what was written
103
+ // meanwhile
104
+ const readOn = (written: number | undefined): void => {
105
+ reading = true
106
+ try {
107
+ let filled = written === undefined || read(written)
108
+ while (filled && connection !== null) {
109
+ const free = inbox.length - held
110
+ filled = read(Number(lib.symbols.tinystore_recv(connection, 0, ptr(inbox, held), free)))
111
+ }
112
+ } finally {
113
+ reading = false
114
+ }
115
+ const next = waiting.shift()
116
+ if (next !== undefined) {
117
+ send(next)
118
+ }
119
+ }
120
+
121
+ const send = (bytes: Uint8Array): void => {
122
+ if (connection === null) {
123
+ return
124
+ }
125
+ if (reading) {
126
+ waiting.push(bytes)
127
+ return
128
+ }
129
+ const free = inbox.length - held
130
+ const written = lib.symbols.tinystore_send(
131
+ connection,
132
+ ptr(bytes),
133
+ bytes.length,
134
+ ptr(inbox, held),
135
+ free,
136
+ )
137
+ readOn(Number(written))
138
+ }
139
+
140
+ const wake = new JSCallback(() => readOn(undefined), {
141
+ args: [FFIType.ptr],
142
+ returns: FFIType.void,
143
+ threadsafe: true,
144
+ })
145
+ const failed = Number(
146
+ lib.symbols.tinystore_open(
147
+ ptr(name),
148
+ name.length,
149
+ ptr(options),
150
+ options.length,
151
+ wake,
152
+ null,
153
+ ptr(opened),
154
+ ptr(failure),
155
+ failure.length,
156
+ ),
157
+ )
158
+ if (failed > 0) {
159
+ const message = new TextDecoder().decode(failure.subarray(0, failed))
160
+ wake.close()
161
+ lib.close()
162
+ throw new ClosedError(`cannot open ${dir}: ${message}`)
163
+ }
164
+ connection = Number(opened[0]) as Pointer
165
+ return {
166
+ write: send,
167
+ close() {
168
+ if (connection === null) {
169
+ return
170
+ }
171
+ const closing = connection
172
+ connection = null
173
+ lib.symbols.tinystore_close(closing)
174
+ wake.close()
175
+ events.end(new ClosedError('the store closed'))
176
+ },
177
+ }
178
+ }
@@ -3,6 +3,7 @@
3
3
  // last lines of a child's stderr.
4
4
 
5
5
  import { spawn } from 'node:child_process'
6
+ import { readdirSync } from 'node:fs'
6
7
 
7
8
  import { ClosedError } from '../errors.ts'
8
9
  import type { Child } from '../runtime.ts'
@@ -59,3 +60,22 @@ export function exitedWith(code: number, stderr: string): ClosedError {
59
60
  `the private server exited with ${code}${stderr === '' ? '' : `: ${lastLines(stderr)}`}`,
60
61
  )
61
62
  }
63
+
64
+ /**
65
+ * The package that carries this platform's executable and libraries:
66
+ * `@tinyshed/tinystore-linux-x64`, or `-musl` after it where Linux's C
67
+ * library is musl, as npm chose it by its `libc` at install.
68
+ */
69
+ export function platformPackage(): string {
70
+ const musl = process.platform === 'linux' && runsOnMusl()
71
+ return `@tinyshed/tinystore-${process.platform}-${process.arch}${musl ? '-musl' : ''}`
72
+ }
73
+
74
+ /** Whether this Linux runs musl: its loader, `ld-musl-<arch>.so.1`, is in /lib. */
75
+ function runsOnMusl(): boolean {
76
+ try {
77
+ return readdirSync('/lib').some(name => name.startsWith('ld-musl-'))
78
+ } catch {
79
+ return false
80
+ }
81
+ }
package/src/runtime.ts CHANGED
@@ -52,6 +52,9 @@ export interface Runtime {
52
52
  readFile(path: string): Promise<Uint8Array | undefined>
53
53
  /** The executable a name finds on PATH. */
54
54
  which(name: string): string | undefined
55
- /** The server binary a package of this runtime carries for this platform. */
56
- packagedBinary(): string | undefined
55
+ /**
56
+ * A file this platform's package carries, `bin/tinystore` or the core's
57
+ * library in `lib/`, installed beside this one; undefined without it.
58
+ */
59
+ packaged(file: string): string | undefined
57
60
  }