@tinyshed/tinystore 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,61 +40,72 @@ export function openAddon(
40
40
  events: AddonEvents,
41
41
  ): Transport {
42
42
  const addon = createRequire(import.meta.url)(path) as Addon
43
+ const reader = new PipeReader(events)
44
+ try {
45
+ reader.pipe = addon.open(dir, options, () => reader.woken())
46
+ } catch (err) {
47
+ throw openFailure(dir, (err as Error).message)
48
+ }
49
+ return {
50
+ write: turning(bytes => reader.send(bytes)),
51
+ close() {
52
+ const closing = reader.pipe
53
+ if (closing === undefined) {
54
+ return
55
+ }
56
+ reader.pipe = undefined
57
+ closing.close()
58
+ events.end(new ClosedError('the store closed'))
59
+ },
60
+ }
61
+ }
43
62
 
63
+ /** The addon's pipe as the session reads and writes it. */
64
+ class PipeReader {
65
+ pipe: AddonPipe | undefined
66
+ readonly #events: AddonEvents
44
67
  // What the session writes while it reads: its frames wait for the read's
45
68
  // end, since an answer handed back meanwhile would reach the session in
46
69
  // the middle of the bytes it is reading.
47
- let reading = false
48
- const waiting: Uint8Array[] = []
70
+ #reading = false
71
+ readonly #waiting: Uint8Array[] = []
49
72
 
50
- const take = (bytes: Uint8Array | null): void => {
51
- reading = true
52
- try {
53
- let next = bytes
54
- while (next !== null && pipe !== undefined) {
55
- events.data(next)
56
- next = pipe.recv()
57
- }
58
- } finally {
59
- reading = false
60
- }
61
- const queued = waiting.shift()
62
- if (queued !== undefined) {
63
- send(queued)
73
+ constructor(events: AddonEvents) {
74
+ this.#events = events
75
+ }
76
+
77
+ /** The core's wake: what it has ready, unless a read under way, which reads until nothing is, takes it. */
78
+ woken(): void {
79
+ if (!this.#reading) {
80
+ this.#take(this.pipe?.recv() ?? null)
64
81
  }
65
82
  }
66
83
 
67
- const send = (bytes: Uint8Array): void => {
68
- if (pipe === undefined) {
84
+ send(bytes: Uint8Array): void {
85
+ if (this.pipe === undefined) {
69
86
  return
70
87
  }
71
- if (reading) {
72
- waiting.push(bytes)
88
+ if (this.#reading) {
89
+ this.#waiting.push(bytes)
73
90
  return
74
91
  }
75
- take(pipe.send(bytes))
92
+ this.#take(this.pipe.send(bytes))
76
93
  }
77
94
 
78
- let pipe: AddonPipe | undefined
79
- try {
80
- pipe = addon.open(dir, options, () => {
81
- if (!reading) {
82
- take(pipe?.recv() ?? null)
83
- }
84
- })
85
- } catch (err) {
86
- throw openFailure(dir, (err as Error).message)
87
- }
88
- return {
89
- write: turning(send),
90
- close() {
91
- const closing = pipe
92
- if (closing === undefined) {
93
- return
95
+ #take(bytes: Uint8Array | null): void {
96
+ this.#reading = true
97
+ try {
98
+ let next = bytes
99
+ while (next !== null && this.pipe !== undefined) {
100
+ this.#events.data(next)
101
+ next = this.pipe.recv()
94
102
  }
95
- pipe = undefined
96
- closing.close()
97
- events.end(new ClosedError('the store closed'))
98
- },
103
+ } finally {
104
+ this.#reading = false
105
+ }
106
+ const queued = this.#waiting.shift()
107
+ if (queued !== undefined) {
108
+ this.send(queued)
109
+ }
99
110
  }
100
111
  }
@@ -4,7 +4,7 @@
4
4
  // memory of the core's comes back: it reads our frames where they are and
5
5
  // writes its own where we say. Bun alone: bun:ffi.
6
6
 
7
- import { dlopen, FFIType, JSCallback, type Pointer, ptr } from 'bun:ffi'
7
+ import { dlopen, FFIType, JSCallback, type Library, type Pointer, ptr } from 'bun:ffi'
8
8
 
9
9
  import { ClosedError } from '../errors.ts'
10
10
  import type { Transport } from '../runtime.ts'
@@ -37,6 +37,8 @@ const symbols = {
37
37
  tinystore_close: { args: [FFIType.ptr], returns: FFIType.void },
38
38
  } as const
39
39
 
40
+ type Core = Library<typeof symbols>
41
+
40
42
  /** What the pipe tells the session reading it. */
41
43
  export interface PipeEvents {
42
44
  /**
@@ -65,144 +67,174 @@ export function openPipe(
65
67
  events: PipeEvents,
66
68
  ): Transport {
67
69
  const lib = dlopen(library, symbols)
70
+ const pipe = new CorePipe(lib, new Inbox(events))
71
+ const wake = new JSCallback(() => pipe.readOn(undefined), {
72
+ args: [FFIType.ptr],
73
+ returns: FFIType.void,
74
+ threadsafe: true,
75
+ })
76
+ pipe.connection = connect(lib, dir, options, wake)
77
+ // Bun's loop with no timer armed spins while it waits for a threadsafe
78
+ // callback from another thread: a write waiting for its sync burned a
79
+ // whole processor (research release-2026-10-23). A timer armed, however
80
+ // far off, lets the loop sleep until the core wakes it; it also keeps the
81
+ // process alive while the store is open, as a sidecar's socket does.
82
+ const sleeps = setInterval(() => {}, 2 ** 30)
83
+ return {
84
+ write: turning(bytes => pipe.send(bytes)),
85
+ close() {
86
+ const closing = pipe.connection
87
+ if (closing === null) {
88
+ return
89
+ }
90
+ pipe.connection = null
91
+ clearInterval(sleeps)
92
+ lib.symbols.tinystore_close(closing)
93
+ wake.close()
94
+ events.end(new ClosedError('the store closed'))
95
+ },
96
+ }
97
+ }
98
+
99
+ /** Opens the store through the core; a store that does not open lets the core and the wake go. */
100
+ function connect(lib: Core, dir: string, options: Uint8Array, wake: JSCallback): Pointer {
68
101
  const name = new TextEncoder().encode(dir)
69
102
  const opened = new BigUint64Array(1)
70
103
  const failure = new Uint8Array(1024)
71
- let connection: Pointer | null = null
104
+ const failed = Number(
105
+ lib.symbols.tinystore_open(
106
+ ptr(name),
107
+ name.length,
108
+ ptr(options),
109
+ options.length,
110
+ wake,
111
+ null,
112
+ ptr(opened),
113
+ ptr(failure),
114
+ failure.length,
115
+ ),
116
+ )
117
+ if (failed > 0) {
118
+ const message = new TextDecoder().decode(failure.subarray(0, failed))
119
+ wake.close()
120
+ lib.close()
121
+ throw openFailure(dir, message)
122
+ }
123
+ return Number(opened[0]) as Pointer
124
+ }
72
125
 
73
- // where the core writes what it owes: whole frames are read from the
74
- // front, and a frame's first part is held there for its rest
75
- const inbox = new Uint8Array(room)
76
- let held = 0
126
+ /**
127
+ * Where the core writes what it owes: whole frames are read from the front,
128
+ * and a frame's first part is held there for its rest.
129
+ */
130
+ class Inbox {
131
+ readonly #events: PipeEvents
132
+ readonly #inbox = new Uint8Array(room)
133
+ #held = 0
77
134
  // A frame longer than the inbox, in bytes as long as it is, and how much
78
135
  // of it came: the core writes its rest there, and the session keeps them,
79
136
  // so that a file's mebibyte is written once here rather than copied out
80
137
  // of an inbox grown for it (issue #10).
81
- let large: Uint8Array | undefined
82
- let filledLarge = 0
83
-
84
- // where the core writes next, and how many bytes it may
85
- const into = (): [Uint8Array, number, number] =>
86
- large === undefined
87
- ? [inbox, held, inbox.length - held]
88
- : [large, filledLarge, large.length - filledLarge]
89
-
90
- // reads what the core wrote where `into` said, and says whether it filled
91
- // that room, which means more may wait
92
- const read = (written: number): boolean => {
93
- if (large !== undefined) {
94
- filledLarge += written
95
- if (filledLarge < large.length) {
138
+ #large: Uint8Array | undefined
139
+ #filledLarge = 0
140
+
141
+ constructor(events: PipeEvents) {
142
+ this.#events = events
143
+ }
144
+
145
+ /** Where the core writes next, and how many bytes it may. */
146
+ into(): [Uint8Array, number, number] {
147
+ return this.#large === undefined
148
+ ? [this.#inbox, this.#held, this.#inbox.length - this.#held]
149
+ : [this.#large, this.#filledLarge, this.#large.length - this.#filledLarge]
150
+ }
151
+
152
+ /**
153
+ * Reads what the core wrote where `into` said, and says whether it filled
154
+ * that room, which means more may wait.
155
+ */
156
+ read(written: number): boolean {
157
+ if (this.#large !== undefined) {
158
+ this.#filledLarge += written
159
+ if (this.#filledLarge < this.#large.length) {
96
160
  return false
97
161
  }
98
- const whole = large
99
- large = undefined
100
- events.frame(whole)
162
+ const whole = this.#large
163
+ this.#large = undefined
164
+ this.#events.frame(whole)
101
165
  return true
102
166
  }
103
- const length = held + written
104
- const filled = length === inbox.length
105
- const taken = written === 0 ? 0 : events.frames(inbox, length)
106
- inbox.copyWithin(0, taken, length)
107
- held = length - taken
108
- const frameLength = held < headerSize ? 0 : headerSize + bodyLength(inbox)
109
- if (frameLength > inbox.length) {
110
- large = new Uint8Array(frameLength)
111
- large.set(inbox.subarray(0, held))
112
- filledLarge = held
113
- held = 0
167
+ const length = this.#held + written
168
+ const filled = length === this.#inbox.length
169
+ const taken = written === 0 ? 0 : this.#events.frames(this.#inbox, length)
170
+ this.#inbox.copyWithin(0, taken, length)
171
+ this.#held = length - taken
172
+ const frameLength = this.#held < headerSize ? 0 : headerSize + bodyLength(this.#inbox)
173
+ if (frameLength > this.#inbox.length) {
174
+ this.#large = new Uint8Array(frameLength)
175
+ this.#large.set(this.#inbox.subarray(0, this.#held))
176
+ this.#filledLarge = this.#held
177
+ this.#held = 0
114
178
  }
115
179
  return filled
116
180
  }
181
+ }
117
182
 
183
+ /** One connection's calls into the core, which write what it owes into the inbox. */
184
+ class CorePipe {
185
+ connection: Pointer | null = null
186
+ readonly #lib: Core
187
+ readonly #inbox: Inbox
118
188
  // What the session writes while it reads: its frames wait for the read's
119
189
  // end, since a call of the core made meanwhile would write over the bytes
120
190
  // being read.
121
- let reading = false
122
- const waiting: Uint8Array[] = []
123
-
124
- // reads what a call of the core wrote, and on while the core fills the
125
- // room, since no wake comes for what waits; then sends what was written
126
- // meanwhile
127
- const readOn = (written: number | undefined): void => {
128
- reading = true
191
+ #reading = false
192
+ readonly #waiting: Uint8Array[] = []
193
+
194
+ constructor(lib: Core, inbox: Inbox) {
195
+ this.#lib = lib
196
+ this.#inbox = inbox
197
+ }
198
+
199
+ /**
200
+ * Reads what a call of the core wrote, and on while the core fills the
201
+ * room, since no wake comes for what waits; then sends what was written
202
+ * meanwhile.
203
+ */
204
+ readOn(written: number | undefined): void {
205
+ this.#reading = true
129
206
  try {
130
- let filled = written === undefined || read(written)
131
- while (filled && connection !== null) {
132
- const [bytes, at, free] = into()
133
- filled = read(Number(lib.symbols.tinystore_recv(connection, 0, ptr(bytes, at), free)))
207
+ let filled = written === undefined || this.#inbox.read(written)
208
+ while (filled && this.connection !== null) {
209
+ const [bytes, at, free] = this.#inbox.into()
210
+ const read = this.#lib.symbols.tinystore_recv(this.connection, 0, ptr(bytes, at), free)
211
+ filled = this.#inbox.read(Number(read))
134
212
  }
135
213
  } finally {
136
- reading = false
214
+ this.#reading = false
137
215
  }
138
- const next = waiting.shift()
216
+ const next = this.#waiting.shift()
139
217
  if (next !== undefined) {
140
- send(next)
218
+ this.send(next)
141
219
  }
142
220
  }
143
221
 
144
- const send = (bytes: Uint8Array): void => {
145
- if (connection === null) {
222
+ send(bytes: Uint8Array): void {
223
+ if (this.connection === null) {
146
224
  return
147
225
  }
148
- if (reading) {
149
- waiting.push(bytes)
226
+ if (this.#reading) {
227
+ this.#waiting.push(bytes)
150
228
  return
151
229
  }
152
- const [owed, at, free] = into()
153
- const written = lib.symbols.tinystore_send(
154
- connection,
230
+ const [owed, at, free] = this.#inbox.into()
231
+ const written = this.#lib.symbols.tinystore_send(
232
+ this.connection,
155
233
  ptr(bytes),
156
234
  bytes.length,
157
235
  ptr(owed, at),
158
236
  free,
159
237
  )
160
- readOn(Number(written))
161
- }
162
-
163
- const wake = new JSCallback(() => readOn(undefined), {
164
- args: [FFIType.ptr],
165
- returns: FFIType.void,
166
- threadsafe: true,
167
- })
168
- const failed = Number(
169
- lib.symbols.tinystore_open(
170
- ptr(name),
171
- name.length,
172
- ptr(options),
173
- options.length,
174
- wake,
175
- null,
176
- ptr(opened),
177
- ptr(failure),
178
- failure.length,
179
- ),
180
- )
181
- if (failed > 0) {
182
- const message = new TextDecoder().decode(failure.subarray(0, failed))
183
- wake.close()
184
- lib.close()
185
- throw openFailure(dir, message)
186
- }
187
- connection = Number(opened[0]) as Pointer
188
- // Bun's loop with no timer armed spins while it waits for a threadsafe
189
- // callback from another thread: a write waiting for its sync burned a
190
- // whole processor (research release-2026-10-23). A timer armed, however
191
- // far off, lets the loop sleep until the core wakes it; it also keeps the
192
- // process alive while the store is open, as a sidecar's socket does.
193
- const sleeps = setInterval(() => {}, 2 ** 30)
194
- return {
195
- write: turning(send),
196
- close() {
197
- if (connection === null) {
198
- return
199
- }
200
- const closing = connection
201
- connection = null
202
- clearInterval(sleeps)
203
- lib.symbols.tinystore_close(closing)
204
- wake.close()
205
- events.end(new ClosedError('the store closed'))
206
- },
238
+ this.readOn(Number(written))
207
239
  }
208
240
  }
package/src/sql.ts CHANGED
@@ -35,6 +35,15 @@ import {
35
35
  } from './jobs.ts'
36
36
  import { Bucket, type BucketOptions, bucketOpen, partsOf, type Values, valuesOf } from './kv.ts'
37
37
  import { Table, type TableOptions } from './query.ts'
38
+ import {
39
+ Appending,
40
+ appendingIn,
41
+ Events,
42
+ type EventsOptions,
43
+ eventsOpen,
44
+ eventsParts,
45
+ TxEvents,
46
+ } from './records.ts'
38
47
  import type { StandardSchemaV1 } from './schema.ts'
39
48
  import { TableIndex, type TableIndexOptions, tableIndexOpen } from './search.ts'
40
49
  import { ConnectionLostError, type Stream } from './session.ts'
@@ -791,6 +800,25 @@ export class Database implements Runner, Home {
791
800
  return new Files(this.#link, name, open, [], this)
792
801
  }
793
802
 
803
+ /**
804
+ * The store's stream of events `stream`, opened through this database:
805
+ * the same calls and reads as `store.events(stream)` gives, the stream in
806
+ * records.db where every stream lives. Its appends inside `db.tx`,
807
+ * through `tx.with(audit)`, and in `db.batch`, through `audit.appending`,
808
+ * are kept in this database's file with the rows and reach the stream
809
+ * once they commit, before the call returns: an event exists exactly when
810
+ * the change it records does.
811
+ *
812
+ * const audit = db.events('audit')
813
+ */
814
+ events(stream: string, options?: EventsOptions): Events {
815
+ const bytes = eventsOpen(stream, options)
816
+ const open = this.keep(methods['records.open'], bytes, async connection =>
817
+ eventsOpen(stream, options, await this.handle(connection)),
818
+ )
819
+ return new Events(this.#link, stream, open, this)
820
+ }
821
+
794
822
  /**
795
823
  * An index of a table of this database, kept in its file: its rows found
796
824
  * again by their words, vectors or both, each hit's value its row as
@@ -911,12 +939,14 @@ export class Database implements Runner, Home {
911
939
  /**
912
940
  * Writes known before they run, written as one: all of them or none, in a
913
941
  * shared commit, and what each did, in their order. A write is a statement,
914
- * or a job's add of a queue opened from the database, `queue.adding`, which
915
- * commits with the rows and changed 1 job, or 0 when its id was taken.
942
+ * a job's add of a queue opened from the database, `queue.adding`, which
943
+ * commits with the rows and changed 1 job, or 0 when its id was taken, or
944
+ * an event's append of a stream opened from it, `audit.appending`, which
945
+ * reaches the stream before the batch returns.
916
946
  *
917
947
  * await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
918
948
  */
919
- async batch(writes: readonly (Sql | Adding)[]): Promise<Done[]> {
949
+ async batch(writes: readonly (Sql | Adding | Appending)[]): Promise<Done[]> {
920
950
  const parts = writes.map(each => {
921
951
  if (each instanceof Sql) {
922
952
  return { text: each.text, values: [...each.values] }
@@ -924,7 +954,12 @@ export class Database implements Runner, Home {
924
954
  if (each instanceof Adding) {
925
955
  return addingIn(each, this)
926
956
  }
927
- throw new InvalidError("a batch takes pieces of SQL, sql`…`, and a queue's adding")
957
+ if (each instanceof Appending) {
958
+ return appendingIn(each, this)
959
+ }
960
+ throw new InvalidError(
961
+ "a batch takes pieces of SQL, sql`…`, a queue's adding and a stream's appending",
962
+ )
928
963
  })
929
964
  this.#refuseAround('a batch')
930
965
  const answered = await this.#link.run('write', async connection => {
@@ -1077,9 +1112,10 @@ export class SqlTx implements Runner {
1077
1112
  with<T>(queue: Queue<T>): TxQueue<T>
1078
1113
  with<T>(channel: Channel<T>): TxChannel<T>
1079
1114
  with(files: Files): Files
1115
+ with(events: Events): TxEvents
1080
1116
  with(
1081
- handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
1082
- ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
1117
+ handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files | Events,
1118
+ ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files | TxEvents {
1083
1119
  const channel = channelParts(handle)
1084
1120
  if (channel !== undefined) {
1085
1121
  // a message published here is handed over once the transaction commits, at the writer
@@ -1103,8 +1139,13 @@ export class SqlTx implements Runner {
1103
1139
  this.#takes(files.home, files.open, `files ${files.name}`)
1104
1140
  return files.inside(this.#via)
1105
1141
  }
1142
+ const events = eventsParts(handle)
1143
+ if (events !== undefined) {
1144
+ this.#takes(events.home, events.open, `events ${events.stream}`)
1145
+ return new TxEvents(events, this.#via)
1146
+ }
1106
1147
  throw new InvalidError(
1107
- `${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
1148
+ `${this.describe}: a transaction takes in a bucket, a queue, files or events opened from its database`,
1108
1149
  )
1109
1150
  }
1110
1151