@tinyshed/tinystore 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +67 -4
- package/dist/config.js +127 -13
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/logger.d.ts +10 -0
- package/dist/logger.js +16 -6
- package/dist/metrics.d.ts +17 -0
- package/dist/metrics.js +22 -1
- package/dist/records.d.ts +90 -5
- package/dist/records.js +114 -6
- package/dist/sql.d.ts +19 -3
- package/dist/sql.js +31 -4
- package/dist/wire/protocol.d.ts +105 -9
- package/dist/wire/protocol.js +296 -8
- package/package.json +8 -8
- package/src/config.ts +204 -25
- package/src/index.ts +13 -1
- package/src/logger.ts +17 -7
- package/src/metrics.ts +25 -0
- package/src/records.ts +178 -7
- package/src/sql.ts +48 -7
- package/src/wire/protocol.ts +321 -8
package/src/records.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import { type Connection, type Link, pieceOf } from './connection.ts'
|
|
9
9
|
import { type ConsoleLine, jsonOf } from './console.ts'
|
|
10
10
|
import { ClosedError, InvalidError } from './errors.ts'
|
|
11
|
-
import { checkName, handleOn, type Via, viaLink } from './handles.ts'
|
|
11
|
+
import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
|
|
12
12
|
import { parseJson } from './json.ts'
|
|
13
13
|
import {
|
|
14
14
|
encodeFields,
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
RecordsDropped,
|
|
35
35
|
type RecordsEvent,
|
|
36
36
|
RecordsFieldNamesAsked,
|
|
37
|
+
RecordsFieldsAsked,
|
|
37
38
|
RecordsNames,
|
|
38
39
|
type RecordsNewEvent,
|
|
39
40
|
RecordsOpen,
|
|
@@ -42,6 +43,7 @@ import {
|
|
|
42
43
|
RecordsPollAsked,
|
|
43
44
|
RecordsQuery,
|
|
44
45
|
RecordsRows,
|
|
46
|
+
RecordsStreamFields,
|
|
45
47
|
RecordsStreams,
|
|
46
48
|
RecordsTailBatch,
|
|
47
49
|
RecordsTopAsked,
|
|
@@ -185,6 +187,31 @@ export interface StreamInfo {
|
|
|
185
187
|
newest: Date
|
|
186
188
|
}
|
|
187
189
|
|
|
190
|
+
/** What a stream's events carry, each by name. */
|
|
191
|
+
export interface StreamFields {
|
|
192
|
+
eventNames: NameCount[]
|
|
193
|
+
fields: FieldInfo[]
|
|
194
|
+
context: FieldInfo[]
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** A name a stream's events have, and how many have it. */
|
|
198
|
+
export interface NameCount {
|
|
199
|
+
name: string
|
|
200
|
+
count: number
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** A key of a stream's events' fields or contexts. */
|
|
204
|
+
export interface FieldInfo {
|
|
205
|
+
name: string
|
|
206
|
+
/** the kinds of value it holds, in the order `ValueKind` lists them */
|
|
207
|
+
kinds: ValueKind[]
|
|
208
|
+
/** the events that carry it: one that writes it twice, once */
|
|
209
|
+
count: number
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** What a value is, as its JSON is spelled: `float` has a fraction or an exponent, `integer` is whole. */
|
|
213
|
+
export type ValueKind = 'float' | 'integer' | 'text' | 'bool' | 'object' | 'list' | 'null'
|
|
214
|
+
|
|
188
215
|
export interface PollBatch {
|
|
189
216
|
events: Event[]
|
|
190
217
|
/** where the next poll begins, kept once these events are safe */
|
|
@@ -253,7 +280,10 @@ abstract class Reads {
|
|
|
253
280
|
/** Keeps a stream that follows the store for its close to cancel; it returns what lets it go. */
|
|
254
281
|
protected abstract follow(stream: Stream): () => void
|
|
255
282
|
|
|
256
|
-
/**
|
|
283
|
+
/**
|
|
284
|
+
* One page: `{ events, next }`, its `limit` of events that match, fewer
|
|
285
|
+
* only on the last, which has no `next`.
|
|
286
|
+
*/
|
|
257
287
|
async list(query?: Query): Promise<Page> {
|
|
258
288
|
const asked = queryOf(query)
|
|
259
289
|
const answer = await this.ask('records.list', handle =>
|
|
@@ -375,25 +405,57 @@ abstract class Reads {
|
|
|
375
405
|
|
|
376
406
|
/** Opens a stream of events: lazily, at its first call on a connection. */
|
|
377
407
|
export function openEvents(link: Link, stream: string, options?: EventsOptions): Events {
|
|
408
|
+
return new Events(link, stream, eventsOpen(stream, options))
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** A stream's open, through the database of `database`, a handle on the connection, when given. */
|
|
412
|
+
export function eventsOpen(stream: string, options?: EventsOptions, database?: number): Uint8Array {
|
|
378
413
|
checkName(stream, 'stream')
|
|
379
414
|
const keep = options?.keep === undefined ? undefined : ms(options.keep)
|
|
380
|
-
return
|
|
415
|
+
return RecordsOpen.encode({ stream, keep, database })
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** What a stream's appends in a database's transaction and batch need of it. */
|
|
419
|
+
interface EventsParts {
|
|
420
|
+
stream: string
|
|
421
|
+
open: Open
|
|
422
|
+
home: Home | undefined
|
|
381
423
|
}
|
|
382
424
|
|
|
425
|
+
let eventsPartsOf: (events: Events) => EventsParts
|
|
426
|
+
|
|
383
427
|
/** A stream of events: written with `append`, which waits for the file, and read back. */
|
|
384
428
|
export class Events extends Reads {
|
|
385
429
|
readonly stream: string
|
|
386
430
|
readonly #link: Link
|
|
387
431
|
readonly #via: Via
|
|
388
|
-
readonly #open:
|
|
432
|
+
readonly #open: Open
|
|
433
|
+
readonly #home: Home | undefined
|
|
389
434
|
|
|
390
|
-
/** Events come from `store.events(stream)
|
|
391
|
-
constructor(link: Link, stream: string, open:
|
|
435
|
+
/** Events come from `store.events(stream)`, or `db.events(stream)` through a database. */
|
|
436
|
+
constructor(link: Link, stream: string, open: Open, home?: Home) {
|
|
392
437
|
super()
|
|
393
438
|
this.stream = stream
|
|
394
439
|
this.#link = link
|
|
395
|
-
this.#via = viaLink(link)
|
|
440
|
+
this.#via = home === undefined ? viaLink(link) : home.via(`events ${stream}`)
|
|
396
441
|
this.#open = open
|
|
442
|
+
this.#home = home
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
static {
|
|
446
|
+
eventsPartsOf = events => ({ stream: events.stream, open: events.#open, home: events.#home })
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* An event's append for a batch of the database the stream was opened
|
|
451
|
+
* from, `db.events`: it is kept with the batch's rows and reaches the
|
|
452
|
+
* stream once they commit, before the batch returns, or never.
|
|
453
|
+
*
|
|
454
|
+
* await db.batch([sql`update roles set role = ${role} where user_id = ${id}`, audit.appending('role.changed', { userId: id, role })])
|
|
455
|
+
*/
|
|
456
|
+
appending(name: string, fields?: Fields, options?: AppendOptions): Appending {
|
|
457
|
+
const event = newEventOf(oneEvent(name, fields, options), 0)
|
|
458
|
+
return new Appending({ ...eventsPartsOf(this), event })
|
|
397
459
|
}
|
|
398
460
|
|
|
399
461
|
/**
|
|
@@ -488,6 +550,92 @@ export class Events extends Reads {
|
|
|
488
550
|
}
|
|
489
551
|
}
|
|
490
552
|
|
|
553
|
+
/** The parts of a stream, when `handle` is one. */
|
|
554
|
+
export function eventsParts(handle: unknown): EventsParts | undefined {
|
|
555
|
+
return handle instanceof Events ? eventsPartsOf(handle) : undefined
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* A stream's appends inside a transaction of the database it was opened
|
|
560
|
+
* from, `tx.with(audit)`: each is kept with the transaction's rows and
|
|
561
|
+
* reaches the stream once it commits, before `db.tx` returns. A read goes
|
|
562
|
+
* through the stream itself, and finds what committed.
|
|
563
|
+
*/
|
|
564
|
+
export class TxEvents {
|
|
565
|
+
readonly stream: string
|
|
566
|
+
readonly #open: Open
|
|
567
|
+
readonly #via: Via
|
|
568
|
+
|
|
569
|
+
/** TxEvents come from `tx.with(events)`. */
|
|
570
|
+
constructor(parts: EventsParts, via: Via) {
|
|
571
|
+
this.stream = parts.stream
|
|
572
|
+
this.#open = parts.open
|
|
573
|
+
this.#via = via
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Appends an event once the transaction commits, and says whether it
|
|
578
|
+
* will be written: false when its stream, or this transaction, holds its id.
|
|
579
|
+
*/
|
|
580
|
+
async append(name: string, fields?: Fields, options?: AppendOptions): Promise<boolean> {
|
|
581
|
+
const event = newEventOf(oneEvent(name, fields, options), 0)
|
|
582
|
+
const answer = await this.#via.call(
|
|
583
|
+
methods['records.open'],
|
|
584
|
+
this.#open,
|
|
585
|
+
methods['records.append'],
|
|
586
|
+
handle => RecordsAppend.encode({ handle, events: [event] }),
|
|
587
|
+
'write',
|
|
588
|
+
)
|
|
589
|
+
return (RecordsAppended.decode(answer).held ?? 0) === 0
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
interface AppendingParts extends EventsParts {
|
|
594
|
+
event: Parameters<typeof RecordsNewEvent.encode>[0]
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
let appendingPartsOf: (appending: Appending) => AppendingParts
|
|
598
|
+
|
|
599
|
+
/** An event's append for a database's batch, from `events.appending`. */
|
|
600
|
+
export class Appending {
|
|
601
|
+
readonly #parts: AppendingParts
|
|
602
|
+
|
|
603
|
+
/** Comes from `events.appending`. */
|
|
604
|
+
constructor(parts: AppendingParts) {
|
|
605
|
+
this.#parts = parts
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
static {
|
|
609
|
+
appendingPartsOf = appending => appending.#parts
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* What a batch of `home` sends of an append, on the connection it goes on:
|
|
615
|
+
* the append's call, its stream's handle opened there first. A stream opened
|
|
616
|
+
* from the store or another database is refused, since its event would not
|
|
617
|
+
* commit with the rows.
|
|
618
|
+
*/
|
|
619
|
+
export function appendingIn(
|
|
620
|
+
appending: Appending,
|
|
621
|
+
home: Home,
|
|
622
|
+
): (connection: Connection) => Promise<{ method: number; body: Uint8Array }> {
|
|
623
|
+
const { stream, open, home: kept, event } = appendingPartsOf(appending)
|
|
624
|
+
if (kept !== home) {
|
|
625
|
+
const where = kept === undefined ? 'records.db' : kept.describe
|
|
626
|
+
throw new InvalidError(
|
|
627
|
+
`${home.describe}: events ${stream}: kept in ${where}, outside this batch of ${home.describe}`,
|
|
628
|
+
)
|
|
629
|
+
}
|
|
630
|
+
return async connection => {
|
|
631
|
+
const handle = await handleOn(connection, methods['records.open'], open)
|
|
632
|
+
return {
|
|
633
|
+
method: methods['records.append'],
|
|
634
|
+
body: RecordsAppend.encode({ handle, events: [event] }),
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
491
639
|
/** Every stream's events, to read: `store.events()`. */
|
|
492
640
|
export class AllEvents extends Reads {
|
|
493
641
|
readonly #link: Link
|
|
@@ -509,6 +657,29 @@ export class AllEvents extends Reads {
|
|
|
509
657
|
}))
|
|
510
658
|
}
|
|
511
659
|
|
|
660
|
+
/**
|
|
661
|
+
* What the events of `stream` carry, none of it read from a packed event: each name with how
|
|
662
|
+
* many events have it, and each key of their fields and contexts with the kinds of value it
|
|
663
|
+
* holds and how many events carry it. A stream with no events carries nothing.
|
|
664
|
+
*/
|
|
665
|
+
async fields(stream: string): Promise<StreamFields> {
|
|
666
|
+
const answer = await this.ask('records.fields', () => RecordsFieldsAsked.encode({ stream }))
|
|
667
|
+
const carried = RecordsStreamFields.decode(answer)
|
|
668
|
+
const info = (field: (typeof carried.fields & {})[number]): FieldInfo => ({
|
|
669
|
+
name: field.name ?? '',
|
|
670
|
+
kinds: (field.kinds ?? []) as ValueKind[],
|
|
671
|
+
count: field.count ?? 0,
|
|
672
|
+
})
|
|
673
|
+
return {
|
|
674
|
+
eventNames: (carried.eventNames ?? []).map(name => ({
|
|
675
|
+
name: name.name ?? '',
|
|
676
|
+
count: name.count ?? 0,
|
|
677
|
+
})),
|
|
678
|
+
fields: (carried.fields ?? []).map(info),
|
|
679
|
+
context: (carried.context ?? []).map(info),
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
512
683
|
/**
|
|
513
684
|
* Every event once, a batch a call, from a cursor the program keeps, the
|
|
514
685
|
* oldest event kept without one: `{ events, cursor, expired }`. With no
|
package/src/sql.ts
CHANGED
|
@@ -35,6 +35,15 @@ import {
|
|
|
35
35
|
} from './jobs.ts'
|
|
36
36
|
import { Bucket, type BucketOptions, bucketOpen, partsOf, type Values, valuesOf } from './kv.ts'
|
|
37
37
|
import { Table, type TableOptions } from './query.ts'
|
|
38
|
+
import {
|
|
39
|
+
Appending,
|
|
40
|
+
appendingIn,
|
|
41
|
+
Events,
|
|
42
|
+
type EventsOptions,
|
|
43
|
+
eventsOpen,
|
|
44
|
+
eventsParts,
|
|
45
|
+
TxEvents,
|
|
46
|
+
} from './records.ts'
|
|
38
47
|
import type { StandardSchemaV1 } from './schema.ts'
|
|
39
48
|
import { TableIndex, type TableIndexOptions, tableIndexOpen } from './search.ts'
|
|
40
49
|
import { ConnectionLostError, type Stream } from './session.ts'
|
|
@@ -791,6 +800,25 @@ export class Database implements Runner, Home {
|
|
|
791
800
|
return new Files(this.#link, name, open, [], this)
|
|
792
801
|
}
|
|
793
802
|
|
|
803
|
+
/**
|
|
804
|
+
* The store's stream of events `stream`, opened through this database:
|
|
805
|
+
* the same calls and reads as `store.events(stream)` gives, the stream in
|
|
806
|
+
* records.db where every stream lives. Its appends inside `db.tx`,
|
|
807
|
+
* through `tx.with(audit)`, and in `db.batch`, through `audit.appending`,
|
|
808
|
+
* are kept in this database's file with the rows and reach the stream
|
|
809
|
+
* once they commit, before the call returns: an event exists exactly when
|
|
810
|
+
* the change it records does.
|
|
811
|
+
*
|
|
812
|
+
* const audit = db.events('audit')
|
|
813
|
+
*/
|
|
814
|
+
events(stream: string, options?: EventsOptions): Events {
|
|
815
|
+
const bytes = eventsOpen(stream, options)
|
|
816
|
+
const open = this.keep(methods['records.open'], bytes, async connection =>
|
|
817
|
+
eventsOpen(stream, options, await this.handle(connection)),
|
|
818
|
+
)
|
|
819
|
+
return new Events(this.#link, stream, open, this)
|
|
820
|
+
}
|
|
821
|
+
|
|
794
822
|
/**
|
|
795
823
|
* An index of a table of this database, kept in its file: its rows found
|
|
796
824
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -911,12 +939,14 @@ export class Database implements Runner, Home {
|
|
|
911
939
|
/**
|
|
912
940
|
* Writes known before they run, written as one: all of them or none, in a
|
|
913
941
|
* shared commit, and what each did, in their order. A write is a statement,
|
|
914
|
-
*
|
|
915
|
-
* commits with the rows and changed 1 job, or 0 when its id was taken
|
|
942
|
+
* a job's add of a queue opened from the database, `queue.adding`, which
|
|
943
|
+
* commits with the rows and changed 1 job, or 0 when its id was taken, or
|
|
944
|
+
* an event's append of a stream opened from it, `audit.appending`, which
|
|
945
|
+
* reaches the stream before the batch returns.
|
|
916
946
|
*
|
|
917
947
|
* await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
|
|
918
948
|
*/
|
|
919
|
-
async batch(writes: readonly (Sql | Adding)[]): Promise<Done[]> {
|
|
949
|
+
async batch(writes: readonly (Sql | Adding | Appending)[]): Promise<Done[]> {
|
|
920
950
|
const parts = writes.map(each => {
|
|
921
951
|
if (each instanceof Sql) {
|
|
922
952
|
return { text: each.text, values: [...each.values] }
|
|
@@ -924,7 +954,12 @@ export class Database implements Runner, Home {
|
|
|
924
954
|
if (each instanceof Adding) {
|
|
925
955
|
return addingIn(each, this)
|
|
926
956
|
}
|
|
927
|
-
|
|
957
|
+
if (each instanceof Appending) {
|
|
958
|
+
return appendingIn(each, this)
|
|
959
|
+
}
|
|
960
|
+
throw new InvalidError(
|
|
961
|
+
"a batch takes pieces of SQL, sql`…`, a queue's adding and a stream's appending",
|
|
962
|
+
)
|
|
928
963
|
})
|
|
929
964
|
this.#refuseAround('a batch')
|
|
930
965
|
const answered = await this.#link.run('write', async connection => {
|
|
@@ -1077,9 +1112,10 @@ export class SqlTx implements Runner {
|
|
|
1077
1112
|
with<T>(queue: Queue<T>): TxQueue<T>
|
|
1078
1113
|
with<T>(channel: Channel<T>): TxChannel<T>
|
|
1079
1114
|
with(files: Files): Files
|
|
1115
|
+
with(events: Events): TxEvents
|
|
1080
1116
|
with(
|
|
1081
|
-
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
|
|
1082
|
-
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
|
|
1117
|
+
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files | Events,
|
|
1118
|
+
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files | TxEvents {
|
|
1083
1119
|
const channel = channelParts(handle)
|
|
1084
1120
|
if (channel !== undefined) {
|
|
1085
1121
|
// a message published here is handed over once the transaction commits, at the writer
|
|
@@ -1103,8 +1139,13 @@ export class SqlTx implements Runner {
|
|
|
1103
1139
|
this.#takes(files.home, files.open, `files ${files.name}`)
|
|
1104
1140
|
return files.inside(this.#via)
|
|
1105
1141
|
}
|
|
1142
|
+
const events = eventsParts(handle)
|
|
1143
|
+
if (events !== undefined) {
|
|
1144
|
+
this.#takes(events.home, events.open, `events ${events.stream}`)
|
|
1145
|
+
return new TxEvents(events, this.#via)
|
|
1146
|
+
}
|
|
1106
1147
|
throw new InvalidError(
|
|
1107
|
-
`${this.describe}: a transaction takes in a bucket, a queue or
|
|
1148
|
+
`${this.describe}: a transaction takes in a bucket, a queue, files or events opened from its database`,
|
|
1108
1149
|
)
|
|
1109
1150
|
}
|
|
1110
1151
|
|