@tinyshed/tinystore 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/backup.d.ts +2 -0
- package/dist/backup.js +1 -0
- package/dist/blobs.d.ts +18 -1
- package/dist/blobs.js +52 -45
- package/dist/console.d.ts +6 -3
- package/dist/console.js +53 -14
- package/dist/directory.d.ts +72 -0
- package/dist/directory.js +121 -0
- package/dist/index.d.ts +2 -1
- package/dist/jobs.d.ts +42 -14
- package/dist/jobs.js +146 -39
- package/dist/locks.js +18 -0
- package/dist/metrics.d.ts +5 -1
- package/dist/metrics.js +2 -0
- package/dist/query.d.ts +8 -1
- package/dist/query.js +56 -19
- package/dist/search.d.ts +17 -1
- package/dist/search.js +11 -2
- package/dist/sql.d.ts +33 -2
- package/dist/sql.js +91 -11
- package/dist/store.d.ts +9 -0
- package/dist/store.js +11 -0
- package/dist/wire/protocol.d.ts +219 -11
- package/dist/wire/protocol.js +650 -18
- package/package.json +8 -8
- package/src/backup.ts +3 -0
- package/src/blobs.ts +66 -45
- package/src/console.ts +56 -16
- package/src/directory.ts +235 -0
- package/src/index.ts +9 -0
- package/src/jobs.ts +187 -46
- package/src/locks.ts +22 -0
- package/src/metrics.ts +7 -1
- package/src/query.ts +68 -24
- package/src/search.ts +28 -5
- package/src/sql.ts +105 -11
- package/src/store.ts +12 -0
- package/src/wire/protocol.ts +685 -18
package/src/jobs.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { FailureLog, messageOf } from './background.ts'
|
|
9
9
|
import type { Connection, Idempotence, Link } from './connection.ts'
|
|
10
|
-
import { CancelledError, CorruptError, InvalidError, UnavailableError } from './errors.ts'
|
|
10
|
+
import { CancelledError, CorruptError, errorOf, InvalidError, UnavailableError } from './errors.ts'
|
|
11
11
|
import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
|
|
12
12
|
import { type Rate, rateOf } from './limits.ts'
|
|
13
13
|
import { sayOwn } from './logger.ts'
|
|
@@ -16,9 +16,11 @@ import { ConnectionLostError, type Stream } from './session.ts'
|
|
|
16
16
|
import { type Duration, ms, type Time, unixMs } from './time.ts'
|
|
17
17
|
import type { Read, Written } from './wire/codec.ts'
|
|
18
18
|
import {
|
|
19
|
+
Handle,
|
|
19
20
|
JobsAnswer,
|
|
20
21
|
JobsCall,
|
|
21
22
|
JobsChanged,
|
|
23
|
+
JobsCounts,
|
|
22
24
|
JobsHeld,
|
|
23
25
|
JobsId,
|
|
24
26
|
JobsJob,
|
|
@@ -124,6 +126,40 @@ export interface ListOptions {
|
|
|
124
126
|
limit?: number | undefined
|
|
125
127
|
}
|
|
126
128
|
|
|
129
|
+
/** How many jobs a queue holds in each state, from one snapshot; a done job has left it. */
|
|
130
|
+
export interface JobCounts {
|
|
131
|
+
/** due later */
|
|
132
|
+
scheduled: number
|
|
133
|
+
/** due, and no handler runs it yet */
|
|
134
|
+
waiting: number
|
|
135
|
+
running: number
|
|
136
|
+
/** kept for the queue's keep */
|
|
137
|
+
failed: number
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Where a job without an id lies, which a cancel names it by: kept on the job
|
|
142
|
+
* a page gave, out of sight, and not on a copy of it.
|
|
143
|
+
*/
|
|
144
|
+
const ROW = Symbol('tinystore.job.row')
|
|
145
|
+
|
|
146
|
+
/** What a cancel sends of a job: its id, or for one without, its row. */
|
|
147
|
+
function namedBy(job: Job<unknown> | string, describe: string): { id?: string; row?: string } {
|
|
148
|
+
if (typeof job === 'string') {
|
|
149
|
+
return { id: job }
|
|
150
|
+
}
|
|
151
|
+
if (job.id !== undefined) {
|
|
152
|
+
return { id: job.id }
|
|
153
|
+
}
|
|
154
|
+
const row = (job as { [ROW]?: string })[ROW]
|
|
155
|
+
if (row === undefined) {
|
|
156
|
+
throw new InvalidError(
|
|
157
|
+
`${describe}: a job without an id names no row: cancel the job a list gave, not a copy`,
|
|
158
|
+
)
|
|
159
|
+
}
|
|
160
|
+
return { row }
|
|
161
|
+
}
|
|
162
|
+
|
|
127
163
|
/** A page of a queue's jobs, and where the next starts when there is one. */
|
|
128
164
|
export interface JobPage<T> {
|
|
129
165
|
jobs: Job<T>[]
|
|
@@ -465,27 +501,115 @@ async function getJob<T>(source: Source<T>, id: string): Promise<Job<T> | undefi
|
|
|
465
501
|
return found.found === true ? jobOf(source, found) : undefined
|
|
466
502
|
}
|
|
467
503
|
|
|
468
|
-
async function cancelJob<T>(source: Source<T>,
|
|
504
|
+
async function cancelJob<T>(source: Source<T>, job: Job<T> | string): Promise<boolean> {
|
|
505
|
+
const named = namedBy(job as Job<unknown> | string, source.describe)
|
|
469
506
|
const body = await callOf(
|
|
470
507
|
source,
|
|
471
508
|
methods['jobs.cancel'],
|
|
472
|
-
handle => JobsId.encode({ handle,
|
|
509
|
+
handle => JobsId.encode({ handle, ...named }),
|
|
473
510
|
'write',
|
|
474
511
|
)
|
|
475
512
|
return JobsChanged.decode(body).changed === true
|
|
476
513
|
}
|
|
477
514
|
|
|
515
|
+
/**
|
|
516
|
+
* A page of the queue's jobs. A walk's page has a job whose value no longer
|
|
517
|
+
* reads as its error in its place; any other page fails on it.
|
|
518
|
+
*/
|
|
519
|
+
async function pageOf<T>(
|
|
520
|
+
source: Source<T>,
|
|
521
|
+
options: ListOptions,
|
|
522
|
+
walk: boolean,
|
|
523
|
+
): Promise<{ jobs: (Job<T> | Error)[]; next: string | undefined }> {
|
|
524
|
+
const body = await callOf(
|
|
525
|
+
source,
|
|
526
|
+
methods['jobs.list'],
|
|
527
|
+
handle => JobsList.encode({ handle, ...options, walk }),
|
|
528
|
+
'read',
|
|
529
|
+
)
|
|
530
|
+
const page = JobsPage.decode(body)
|
|
531
|
+
const jobs: (Job<T> | Error)[] = []
|
|
532
|
+
for (const found of page.jobs ?? []) {
|
|
533
|
+
if (found.failure !== undefined) {
|
|
534
|
+
const { code, message, what } = found.failure
|
|
535
|
+
jobs.push(errorOf(code ?? 'internal', message ?? '', what ?? {}))
|
|
536
|
+
continue
|
|
537
|
+
}
|
|
538
|
+
const job = jobOf(source, found)
|
|
539
|
+
jobs.push(walk ? await job.catch((err: Error) => err) : await job)
|
|
540
|
+
}
|
|
541
|
+
return { jobs, next: page.next }
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Every job a filter names, a page at a time. A job whose value no longer
|
|
546
|
+
* reads is its error in its place: `next()` rejects with it, and the call
|
|
547
|
+
* after goes on past it, as a `for await` that catches it and iterates the
|
|
548
|
+
* same walk again does.
|
|
549
|
+
*/
|
|
550
|
+
class Walk<T> implements AsyncGenerator<Job<T>, void, unknown> {
|
|
551
|
+
readonly #source: Source<T>
|
|
552
|
+
readonly #options: Omit<ListOptions, 'after'>
|
|
553
|
+
#jobs: (Job<T> | Error)[] = []
|
|
554
|
+
#after: string | undefined
|
|
555
|
+
#ended = false
|
|
556
|
+
|
|
557
|
+
constructor(source: Source<T>, options: Omit<ListOptions, 'after'>) {
|
|
558
|
+
this.#source = source
|
|
559
|
+
this.#options = options
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
async next(): Promise<IteratorResult<Job<T>, void>> {
|
|
563
|
+
while (this.#jobs.length === 0 && !this.#ended) {
|
|
564
|
+
const page = await pageOf(this.#source, { ...this.#options, after: this.#after }, true)
|
|
565
|
+
this.#jobs = page.jobs
|
|
566
|
+
this.#after = page.next
|
|
567
|
+
this.#ended = page.next === undefined
|
|
568
|
+
}
|
|
569
|
+
const job = this.#jobs.shift()
|
|
570
|
+
if (job === undefined) {
|
|
571
|
+
return { done: true, value: undefined }
|
|
572
|
+
}
|
|
573
|
+
if (job instanceof Error) {
|
|
574
|
+
throw job
|
|
575
|
+
}
|
|
576
|
+
return { done: false, value: job }
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
async return(): Promise<IteratorResult<Job<T>, void>> {
|
|
580
|
+
this.#end()
|
|
581
|
+
return { done: true, value: undefined }
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
async throw(err: unknown): Promise<IteratorResult<Job<T>, void>> {
|
|
585
|
+
this.#end()
|
|
586
|
+
throw err
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
[Symbol.asyncIterator](): this {
|
|
590
|
+
return this
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
594
|
+
this.#end()
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
#end(): void {
|
|
598
|
+
this.#jobs = []
|
|
599
|
+
this.#ended = true
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
478
603
|
/** A job as the server sent it, its value read as the queue reads one. */
|
|
479
604
|
async function jobOf<T>(source: Source<T>, found: Read<typeof JobsJob.fields>): Promise<Job<T>> {
|
|
480
605
|
let value: T
|
|
481
606
|
try {
|
|
482
607
|
value = await source.values.decode(found.value ?? 'null')
|
|
483
608
|
} catch (err) {
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
)
|
|
609
|
+
const named = found.id === undefined ? 'a job without an id' : `id ${JSON.stringify(found.id)}`
|
|
610
|
+
throw new CorruptError(`${source.describe}: ${named}: ${messageOf(err)}`)
|
|
487
611
|
}
|
|
488
|
-
|
|
612
|
+
const job: Job<T> = {
|
|
489
613
|
id: found.id,
|
|
490
614
|
value,
|
|
491
615
|
state: (found.state ?? 'waiting') as JobState,
|
|
@@ -504,6 +628,10 @@ async function jobOf<T>(source: Source<T>, found: Read<typeof JobsJob.fields>):
|
|
|
504
628
|
endedAt: new Date(found.endedAt ?? found.startedAt),
|
|
505
629
|
},
|
|
506
630
|
}
|
|
631
|
+
if (found.row !== undefined) {
|
|
632
|
+
Object.defineProperty(job, ROW, { value: found.row })
|
|
633
|
+
}
|
|
634
|
+
return job
|
|
507
635
|
}
|
|
508
636
|
|
|
509
637
|
/**
|
|
@@ -639,9 +767,10 @@ export class Queue<T> {
|
|
|
639
767
|
}
|
|
640
768
|
|
|
641
769
|
/**
|
|
642
|
-
* Changes the id's job when it
|
|
643
|
-
*
|
|
644
|
-
* job that runs,
|
|
770
|
+
* Changes the id's job when it is not running: its value, the steps kept
|
|
771
|
+
* for the old one dropped with it, and its time, group or repeat when the
|
|
772
|
+
* options give them. It says whether it did: a job that runs, is done or
|
|
773
|
+
* failed, or never was answers false.
|
|
645
774
|
*/
|
|
646
775
|
async update(id: string, value: T, options: JobOptions = {}): Promise<boolean> {
|
|
647
776
|
const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) }
|
|
@@ -649,12 +778,34 @@ export class Queue<T> {
|
|
|
649
778
|
}
|
|
650
779
|
|
|
651
780
|
/**
|
|
652
|
-
* Takes
|
|
653
|
-
*
|
|
781
|
+
* Takes a job, whatever its state, and says whether it was there: the job
|
|
782
|
+
* under an id, or a job a list gave, by its id or, for one without, by the
|
|
783
|
+
* row it was listed from. A job retried, run again for its repeat or
|
|
784
|
+
* cancelled since is in no row, and its cancel takes nothing, never another
|
|
785
|
+
* job. A running one's handler sees its signal abort, and what it returns
|
|
654
786
|
* then settles nothing; a repeating one stops.
|
|
787
|
+
*
|
|
788
|
+
* for await (const job of emails.all({ state: 'scheduled' })) if (job.value.to === bounced) await emails.cancel(job)
|
|
655
789
|
*/
|
|
656
|
-
cancel(
|
|
657
|
-
return cancelJob(this.#source,
|
|
790
|
+
cancel(job: Job<T> | string): Promise<boolean> {
|
|
791
|
+
return cancelJob(this.#source, job)
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/** How many jobs the queue holds in each state, from one snapshot; it reads no value. */
|
|
795
|
+
async counts(): Promise<JobCounts> {
|
|
796
|
+
const body = await callOf(
|
|
797
|
+
this.#source,
|
|
798
|
+
methods['jobs.counts'],
|
|
799
|
+
handle => Handle.encode({ handle }),
|
|
800
|
+
'read',
|
|
801
|
+
)
|
|
802
|
+
const { scheduled, waiting, running, failed } = JobsCounts.decode(body)
|
|
803
|
+
return {
|
|
804
|
+
scheduled: scheduled ?? 0,
|
|
805
|
+
waiting: waiting ?? 0,
|
|
806
|
+
running: running ?? 0,
|
|
807
|
+
failed: failed ?? 0,
|
|
808
|
+
}
|
|
658
809
|
}
|
|
659
810
|
|
|
660
811
|
/**
|
|
@@ -679,33 +830,22 @@ export class Queue<T> {
|
|
|
679
830
|
return watchJob(this.#source, id)
|
|
680
831
|
}
|
|
681
832
|
|
|
682
|
-
/**
|
|
833
|
+
/**
|
|
834
|
+
* A page of the queue's jobs: with no prefix every job, those with ids by id,
|
|
835
|
+
* then those without as they run, then those without that failed.
|
|
836
|
+
*/
|
|
683
837
|
async list(options: ListOptions = {}): Promise<JobPage<T>> {
|
|
684
|
-
const
|
|
685
|
-
|
|
686
|
-
methods['jobs.list'],
|
|
687
|
-
handle => JobsList.encode({ handle, ...options }),
|
|
688
|
-
'read',
|
|
689
|
-
)
|
|
690
|
-
const page = JobsPage.decode(body)
|
|
691
|
-
const jobs: Job<T>[] = []
|
|
692
|
-
for (const found of page.jobs ?? []) {
|
|
693
|
-
jobs.push(await jobOf(this.#source, found))
|
|
694
|
-
}
|
|
695
|
-
return { jobs, next: page.next }
|
|
838
|
+
const page = await pageOf(this.#source, options, false)
|
|
839
|
+
return { jobs: page.jobs as Job<T>[], next: page.next }
|
|
696
840
|
}
|
|
697
841
|
|
|
698
|
-
/**
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
return
|
|
706
|
-
}
|
|
707
|
-
after = page.next
|
|
708
|
-
}
|
|
842
|
+
/**
|
|
843
|
+
* Every job the options name, a page at a time, holding nothing between
|
|
844
|
+
* pages. A job whose value no longer reads is a CorruptError in its place,
|
|
845
|
+
* naming it, and iterating the same walk again goes on past it.
|
|
846
|
+
*/
|
|
847
|
+
all(options: Omit<ListOptions, 'after'> = {}): AsyncGenerator<Job<T>> {
|
|
848
|
+
return new Walk(this.#source, options)
|
|
709
849
|
}
|
|
710
850
|
|
|
711
851
|
/**
|
|
@@ -776,8 +916,8 @@ export class TxQueue<T> {
|
|
|
776
916
|
return this.#queue.update(id, value, options)
|
|
777
917
|
}
|
|
778
918
|
|
|
779
|
-
cancel(
|
|
780
|
-
return this.#queue.cancel(
|
|
919
|
+
cancel(job: Job<T> | string): Promise<boolean> {
|
|
920
|
+
return this.#queue.cancel(job)
|
|
781
921
|
}
|
|
782
922
|
|
|
783
923
|
/** The job under id as the transaction sees it, its own writes among it. */
|
|
@@ -832,12 +972,12 @@ export function addingIn(
|
|
|
832
972
|
}
|
|
833
973
|
}
|
|
834
974
|
|
|
835
|
-
/** A write the store's transaction keeps for its commit: its method, and its call or the
|
|
975
|
+
/** A write the store's transaction keeps for its commit: its method, and its call or the job it cancels. */
|
|
836
976
|
export interface JobOp {
|
|
837
977
|
source: Source<unknown>
|
|
838
978
|
method: number
|
|
839
979
|
call?: Omit<Written<typeof JobsCall.fields>, 'handle'>
|
|
840
|
-
id?: string
|
|
980
|
+
id?: { id?: string; row?: string }
|
|
841
981
|
}
|
|
842
982
|
|
|
843
983
|
/**
|
|
@@ -871,13 +1011,14 @@ export class StoreTxQueue<T> {
|
|
|
871
1011
|
this.#write('jobs.set', { ...jobFields(options), id }, value)
|
|
872
1012
|
}
|
|
873
1013
|
|
|
874
|
-
/** Changes the id's job when it
|
|
1014
|
+
/** Changes the id's job when it is not running, its old value's steps dropped, with the commit. */
|
|
875
1015
|
async update(id: string, value: T, options: JobOptions = {}): Promise<void> {
|
|
876
1016
|
this.#write('jobs.update', { ...jobFields(options), id }, value)
|
|
877
1017
|
}
|
|
878
1018
|
|
|
879
|
-
/** Takes the job under id, whatever its state, with the commit. */
|
|
880
|
-
async cancel(
|
|
1019
|
+
/** Takes the job under id, or a job a list gave, whatever its state, with the commit. */
|
|
1020
|
+
async cancel(job: Job<T> | string): Promise<void> {
|
|
1021
|
+
const id = namedBy(job as Job<unknown> | string, this.#source.describe)
|
|
881
1022
|
this.#keep({ source: this.#source as Source<unknown>, method: methods['jobs.cancel'], id })
|
|
882
1023
|
}
|
|
883
1024
|
|
|
@@ -900,7 +1041,7 @@ export async function commitJobs(link: Link, ops: readonly JobOp[]): Promise<voi
|
|
|
900
1041
|
writes.push({
|
|
901
1042
|
method: op.method,
|
|
902
1043
|
call: op.call === undefined ? undefined : { ...op.call, handle },
|
|
903
|
-
id: op.id === undefined ? undefined : { handle,
|
|
1044
|
+
id: op.id === undefined ? undefined : { handle, ...op.id },
|
|
904
1045
|
})
|
|
905
1046
|
}
|
|
906
1047
|
await connection.session.call(methods['jobs.tx'], JobsTx.encode({ writes }))
|
package/src/locks.ts
CHANGED
|
@@ -96,6 +96,7 @@ export class Semaphore {
|
|
|
96
96
|
if (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 1000) {
|
|
97
97
|
throw new InvalidError(`semaphore ${name}: a limit of ${options.limit}, not 1 to 1000`)
|
|
98
98
|
}
|
|
99
|
+
keepLimit(link, name, options.limit)
|
|
99
100
|
this.name = name
|
|
100
101
|
this.limit = options.limit
|
|
101
102
|
this.#keys = new Keys(link, 'semaphore', name, options.limit, options)
|
|
@@ -114,6 +115,27 @@ export class Semaphore {
|
|
|
114
115
|
}
|
|
115
116
|
}
|
|
116
117
|
|
|
118
|
+
// The limit each semaphore's first open gave, by the link of the store that
|
|
119
|
+
// opened it: one name keeps one limit in a program, and two stores are two
|
|
120
|
+
// programs, each of whose holds the store admits by its own limit.
|
|
121
|
+
const opened = new WeakMap<Link, Map<string, number>>()
|
|
122
|
+
|
|
123
|
+
/** Keeps the limit a semaphore's first open on this store gives: another limit is refused, naming both. */
|
|
124
|
+
function keepLimit(link: Link, name: string, limit: number): void {
|
|
125
|
+
let limits = opened.get(link)
|
|
126
|
+
if (limits === undefined) {
|
|
127
|
+
limits = new Map()
|
|
128
|
+
opened.set(link, limits)
|
|
129
|
+
}
|
|
130
|
+
const first = limits.get(name) ?? limit
|
|
131
|
+
if (first !== limit) {
|
|
132
|
+
throw new InvalidError(
|
|
133
|
+
`semaphore ${name}: open in this program with a limit of ${first}, not ${limit}`,
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
limits.set(name, first)
|
|
137
|
+
}
|
|
138
|
+
|
|
117
139
|
function waitOf(options: HoldOptions | undefined): number {
|
|
118
140
|
return ms(options?.wait ?? defaultWait)
|
|
119
141
|
}
|
package/src/metrics.ts
CHANGED
|
@@ -185,6 +185,8 @@ export interface AggregateQuery<O extends Op = Op> extends Query {
|
|
|
185
185
|
without?: readonly string[] | undefined
|
|
186
186
|
/** a timer's percentile also gives a point between its bounds, interpolated: an estimate */
|
|
187
187
|
estimate?: boolean | undefined
|
|
188
|
+
/** a NaN sample counts in no bucket, which says how many it passed over in skipped; an infinity is still refused */
|
|
189
|
+
skipNaN?: boolean | undefined
|
|
188
190
|
}
|
|
189
191
|
|
|
190
192
|
/** What every bucket of an aggregate says of itself. */
|
|
@@ -194,6 +196,8 @@ interface Spanned {
|
|
|
194
196
|
/** how many samples it comes from; a timer's, how many durations */
|
|
195
197
|
samples: number
|
|
196
198
|
resets: number
|
|
199
|
+
/** the NaN samples it passed over, with skipNaN */
|
|
200
|
+
skipped: number
|
|
197
201
|
/** its value comes from less than its span: the range, the clock or retention cut it */
|
|
198
202
|
partial: boolean
|
|
199
203
|
/** its first step began before the bucket before it: a whole bucket between had no samples */
|
|
@@ -225,7 +229,7 @@ export interface Aggregate<O extends Op = Op> {
|
|
|
225
229
|
name: string
|
|
226
230
|
/** the labels by kept or without left */
|
|
227
231
|
labels: Labels
|
|
228
|
-
/** a bucket without samples is left out */
|
|
232
|
+
/** a bucket without samples is left out, and so is one of NaN alone */
|
|
229
233
|
buckets: Bucket<O>[]
|
|
230
234
|
/** a page cut the group here: the page its next reads begins with the rest of its buckets */
|
|
231
235
|
more: boolean
|
|
@@ -754,6 +758,7 @@ function aggregateOf(query: AggregateQuery): Parameters<typeof MetricsAggregateA
|
|
|
754
758
|
by: query.by === undefined ? undefined : [...query.by],
|
|
755
759
|
without: query.without === undefined ? undefined : [...query.without],
|
|
756
760
|
estimate: query.estimate === true,
|
|
761
|
+
skipNan: query.skipNaN === true,
|
|
757
762
|
}
|
|
758
763
|
}
|
|
759
764
|
|
|
@@ -771,6 +776,7 @@ function bucketOf(bucket: WireBucket): PercentileBucket {
|
|
|
771
776
|
at: new Date(bucket.at ?? 0),
|
|
772
777
|
samples: bucket.samples ?? 0,
|
|
773
778
|
resets: bucket.resets ?? 0,
|
|
779
|
+
skipped: bucket.skipped ?? 0,
|
|
774
780
|
partial: bucket.partial === true,
|
|
775
781
|
gap: bucket.gap === true,
|
|
776
782
|
}
|
package/src/query.ts
CHANGED
|
@@ -8,18 +8,21 @@ import { check, type StandardSchemaV1 } from './schema.ts'
|
|
|
8
8
|
import {
|
|
9
9
|
type Condition,
|
|
10
10
|
cellOf,
|
|
11
|
+
checked,
|
|
11
12
|
conditionOf,
|
|
12
13
|
type Done,
|
|
13
14
|
type Embeddable,
|
|
14
15
|
grouped,
|
|
15
16
|
identOf,
|
|
16
17
|
joinedWith,
|
|
18
|
+
type Ordered,
|
|
17
19
|
placeholders,
|
|
18
20
|
type Row,
|
|
19
21
|
type Runner,
|
|
20
22
|
Sql,
|
|
21
23
|
type SqlArg,
|
|
22
24
|
type SqlValue,
|
|
25
|
+
walk,
|
|
23
26
|
} from './sql.ts'
|
|
24
27
|
|
|
25
28
|
/** How a column SQLite keeps otherwise reads in JavaScript. */
|
|
@@ -120,7 +123,7 @@ export class Query<T = Row> implements Embeddable {
|
|
|
120
123
|
|
|
121
124
|
/** The columns a row holds, as SQL text, and the shape of the row they make. */
|
|
122
125
|
select<R = Row>(columns: string | Sql): Query<R> {
|
|
123
|
-
const select = typeof columns === 'string' ? new Sql(columns, []) : columns
|
|
126
|
+
const select = checked(typeof columns === 'string' ? new Sql(columns, []) : columns)
|
|
124
127
|
return new Query<R>(this.#source, { ...this.#parts, select })
|
|
125
128
|
}
|
|
126
129
|
|
|
@@ -193,7 +196,7 @@ export class Query<T = Row> implements Embeddable {
|
|
|
193
196
|
}
|
|
194
197
|
const order =
|
|
195
198
|
column instanceof Sql
|
|
196
|
-
? { piece: column, direction }
|
|
199
|
+
? { piece: checked(column), direction }
|
|
197
200
|
: { piece: new Sql(identOf(column), []), column, direction }
|
|
198
201
|
return this.#with({ order: [...this.#parts.order, order] })
|
|
199
202
|
}
|
|
@@ -254,16 +257,21 @@ export class Query<T = Row> implements Embeddable {
|
|
|
254
257
|
return Number(answered.rows[0]?.[0] ?? 0)
|
|
255
258
|
}
|
|
256
259
|
|
|
257
|
-
/**
|
|
260
|
+
/**
|
|
261
|
+
* The rows one at a time, as their parts arrive: for more than fits in
|
|
262
|
+
* memory. The server reads them from one snapshot of five seconds at
|
|
263
|
+
* most; ordered by columns by their names, a unique one last, as `list`
|
|
264
|
+
* is, it reads on past that after the last row it sent, and any other
|
|
265
|
+
* walk fails `limit` there. Inside a transaction it reads the rows `all`
|
|
266
|
+
* does.
|
|
267
|
+
*/
|
|
258
268
|
async *each(): AsyncGenerator<T> {
|
|
259
|
-
const runner = this.#source.runner
|
|
260
|
-
|
|
261
|
-
}
|
|
262
|
-
if (runner.each === undefined) {
|
|
269
|
+
const runner = this.#source.runner
|
|
270
|
+
if (runner[walk] === undefined) {
|
|
263
271
|
yield* await this.all()
|
|
264
272
|
return
|
|
265
273
|
}
|
|
266
|
-
for await (const row of runner
|
|
274
|
+
for await (const row of runner[walk]<Row>(render(this.#parts), walkOrder(this.#parts.order))) {
|
|
267
275
|
yield (await this.#rows(Object.keys(row), [Object.values(row)]))[0]!
|
|
268
276
|
}
|
|
269
277
|
}
|
|
@@ -285,9 +293,16 @@ export class Query<T = Row> implements Embeddable {
|
|
|
285
293
|
'a page is read by the query order, of columns by their names: orderBy(…) first',
|
|
286
294
|
)
|
|
287
295
|
}
|
|
296
|
+
// A page reads its limit's rows after its cursor: the query's own limit or
|
|
297
|
+
// offset would be dropped, and limit(100) page through every row.
|
|
298
|
+
if (this.#parts.limit !== undefined || this.#parts.offset !== undefined) {
|
|
299
|
+
throw new InvalidError(
|
|
300
|
+
'list after a limit or an offset: list pages the rows itself and takes its own limit',
|
|
301
|
+
)
|
|
302
|
+
}
|
|
288
303
|
const limit = whole(options.limit ?? 100, 'a page limit')
|
|
289
|
-
const digest = digestOf(render(
|
|
290
|
-
let parts: Parts = { ...this.#parts, limit
|
|
304
|
+
const digest = digestOf(render(this.#parts))
|
|
305
|
+
let parts: Parts = { ...this.#parts, limit }
|
|
291
306
|
if (options.after !== undefined) {
|
|
292
307
|
const after = cursorOf(options.after, digest)
|
|
293
308
|
parts = {
|
|
@@ -337,8 +352,9 @@ export class Query<T = Row> implements Embeddable {
|
|
|
337
352
|
for (const name of names) {
|
|
338
353
|
const value = values[name]
|
|
339
354
|
if (value instanceof Sql) {
|
|
340
|
-
|
|
341
|
-
|
|
355
|
+
const set = checked(value)
|
|
356
|
+
sets.push(`${identOf(name)} = ${set.text}`)
|
|
357
|
+
out.push(...set.values)
|
|
342
358
|
} else {
|
|
343
359
|
sets.push(`${identOf(name)} = ?`)
|
|
344
360
|
out.push(fieldOf(value, name, this.#source.types))
|
|
@@ -399,7 +415,7 @@ export class Query<T = Row> implements Embeddable {
|
|
|
399
415
|
}
|
|
400
416
|
|
|
401
417
|
#joined(kind: string, table: string, on: string | Sql): Query<T> {
|
|
402
|
-
const condition = typeof on === 'string' ? new Sql(on, []) : on
|
|
418
|
+
const condition = checked(typeof on === 'string' ? new Sql(on, []) : on)
|
|
403
419
|
const ref = tableRef(table)
|
|
404
420
|
const join = new Sql(`${kind} ${ref.text} on ${condition.text}`, condition.values)
|
|
405
421
|
return this.#with({ joins: [...this.#parts.joins, join] })
|
|
@@ -414,6 +430,22 @@ export class Query<T = Row> implements Embeddable {
|
|
|
414
430
|
if (this.#parts.joins.length > 0) {
|
|
415
431
|
throw new InvalidError(`${what} of a join: write it in SQL`)
|
|
416
432
|
}
|
|
433
|
+
// SQLite's update and delete take none of these: limit(10).delete()
|
|
434
|
+
// would delete every row the conditions match.
|
|
435
|
+
const parts = this.#parts
|
|
436
|
+
const shaped =
|
|
437
|
+
parts.limit !== undefined ||
|
|
438
|
+
parts.offset !== undefined ||
|
|
439
|
+
parts.order.length > 0 ||
|
|
440
|
+
parts.groups.length > 0 ||
|
|
441
|
+
parts.having.length > 0 ||
|
|
442
|
+
parts.select !== undefined ||
|
|
443
|
+
parts.includes.length > 0
|
|
444
|
+
if (shaped) {
|
|
445
|
+
throw new InvalidError(
|
|
446
|
+
`${what} with a limit, offset, order, group, select or include: narrow it by conditions, or write it in SQL`,
|
|
447
|
+
)
|
|
448
|
+
}
|
|
417
449
|
}
|
|
418
450
|
}
|
|
419
451
|
|
|
@@ -484,7 +516,7 @@ export class Table<T = Row> extends Query<T> {
|
|
|
484
516
|
const values = [...insert.values]
|
|
485
517
|
if (options.where !== undefined) {
|
|
486
518
|
// in an upsert's update, a column is the row's that holds the key
|
|
487
|
-
const where = conditionOf(options.where)
|
|
519
|
+
const where = checked(conditionOf(options.where))
|
|
488
520
|
text += ` where ${where.text}`
|
|
489
521
|
values.push(...where.values)
|
|
490
522
|
}
|
|
@@ -786,21 +818,18 @@ function kept(condition: Condition | string, values: unknown[]): Kept[] {
|
|
|
786
818
|
`${counted.positional} placeholders and ${values.length} values: each ? takes one value`,
|
|
787
819
|
)
|
|
788
820
|
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
),
|
|
795
|
-
bare: false,
|
|
796
|
-
},
|
|
797
|
-
]
|
|
821
|
+
const piece = new Sql(
|
|
822
|
+
condition,
|
|
823
|
+
values.map((value, at) => cellOf(value, `value ${at + 1}`)),
|
|
824
|
+
)
|
|
825
|
+
return [{ piece: checked(piece), bare: false }]
|
|
798
826
|
}
|
|
799
827
|
if (values.length > 0) {
|
|
800
828
|
throw new InvalidError('a condition that is not text carries its values: pass it alone')
|
|
801
829
|
}
|
|
802
830
|
if (condition instanceof Sql) {
|
|
803
|
-
|
|
831
|
+
const piece = checked(condition)
|
|
832
|
+
return [{ piece, bare: piece.bare || grouped(piece.text) }]
|
|
804
833
|
}
|
|
805
834
|
return Object.keys(condition)
|
|
806
835
|
.sort()
|
|
@@ -868,6 +897,21 @@ function render(parts: Parts): Sql {
|
|
|
868
897
|
return new Sql(text, values)
|
|
869
898
|
}
|
|
870
899
|
|
|
900
|
+
/**
|
|
901
|
+
* The columns a walk's rows are ordered by, as its rows name them, when each
|
|
902
|
+
* is a column by its name: the server reads on past one snapshot by them, as
|
|
903
|
+
* `list` pages by them.
|
|
904
|
+
*/
|
|
905
|
+
function walkOrder(order: Parts['order']): Ordered[] | undefined {
|
|
906
|
+
if (order.length === 0 || order.some(each => each.column === undefined)) {
|
|
907
|
+
return undefined
|
|
908
|
+
}
|
|
909
|
+
return order.map(each => ({
|
|
910
|
+
column: each.column!.split('.').pop()!,
|
|
911
|
+
descending: each.direction === 'desc',
|
|
912
|
+
}))
|
|
913
|
+
}
|
|
914
|
+
|
|
871
915
|
/**
|
|
872
916
|
* The condition that starts a page after a row's order values. SQLite orders
|
|
873
917
|
* a null before every value, first in ascending order and last in descending,
|
package/src/search.ts
CHANGED
|
@@ -94,6 +94,13 @@ export interface Query {
|
|
|
94
94
|
where?: Where | undefined
|
|
95
95
|
/** this search's way of combining its words and its vector, in place of the index's */
|
|
96
96
|
hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
|
|
97
|
+
/**
|
|
98
|
+
* the least cosine, from -1 to 1, a document's vector has with the
|
|
99
|
+
* search's for the vector to find it: a search by its vector alone leaves
|
|
100
|
+
* out the hits under it, and one by words too adds nothing for them to
|
|
101
|
+
* the sum; invalid beside a rerank
|
|
102
|
+
*/
|
|
103
|
+
minSimilarity?: number | undefined
|
|
97
104
|
/**
|
|
98
105
|
* the last word read as the start of a word too, as a search box types:
|
|
99
106
|
* `crypt` finds `cryptography` as well, at half its field's weight; a
|
|
@@ -137,6 +144,10 @@ export interface Hit<T> {
|
|
|
137
144
|
score: number
|
|
138
145
|
/** where the search's words matched it, when the search asked with `matches` */
|
|
139
146
|
matches?: Match[]
|
|
147
|
+
/** the score its words gave it, as a search by its words alone scores it, where they found it among their best */
|
|
148
|
+
textScore?: number
|
|
149
|
+
/** the cosine of its vector with the search's, from -1 to 1, where both have one */
|
|
150
|
+
similarity?: number
|
|
140
151
|
}
|
|
141
152
|
|
|
142
153
|
/** What a search found, the best first, and the text it read in fact when a word was misspelt. */
|
|
@@ -145,10 +156,13 @@ export interface Found<T> {
|
|
|
145
156
|
corrected?: string | undefined
|
|
146
157
|
}
|
|
147
158
|
|
|
148
|
-
/**
|
|
159
|
+
/**
|
|
160
|
+
* Where an index stands: serving what it was opened with, read again beside it, or a rebuild that failed.
|
|
161
|
+
* `error` beside `ready` and `rebuilding` is the last maintenance pass's failure while the passes keep failing.
|
|
162
|
+
*/
|
|
149
163
|
export type IndexStatus =
|
|
150
|
-
| { state: 'ready' }
|
|
151
|
-
| { state: 'rebuilding'; done: number }
|
|
164
|
+
| { state: 'ready'; error?: string | undefined }
|
|
165
|
+
| { state: 'rebuilding'; done: number; error?: string | undefined }
|
|
152
166
|
| { state: 'failed'; error: string }
|
|
153
167
|
|
|
154
168
|
/** An index as `store.indexes()` lists it. */
|
|
@@ -230,6 +244,7 @@ abstract class Searchable<T> {
|
|
|
230
244
|
vector,
|
|
231
245
|
vectorWeight: asked.hybrid?.vectorWeight,
|
|
232
246
|
rerank: asked.hybrid?.rerank,
|
|
247
|
+
minSimilarity: asked.minSimilarity,
|
|
233
248
|
prefix: asked.prefix === true || undefined,
|
|
234
249
|
// none unless asked, so that a server before it reads the search
|
|
235
250
|
prefixEvery: asked.prefix === 'every' || undefined,
|
|
@@ -251,6 +266,12 @@ abstract class Searchable<T> {
|
|
|
251
266
|
if (hit.matches !== undefined) {
|
|
252
267
|
read.matches = hit.matches.map(matchOf)
|
|
253
268
|
}
|
|
269
|
+
if (hit.textScore !== undefined) {
|
|
270
|
+
read.textScore = hit.textScore
|
|
271
|
+
}
|
|
272
|
+
if (hit.similarity !== undefined) {
|
|
273
|
+
read.similarity = hit.similarity
|
|
274
|
+
}
|
|
254
275
|
return read
|
|
255
276
|
})
|
|
256
277
|
return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected }
|
|
@@ -445,13 +466,15 @@ function statusOf(status: {
|
|
|
445
466
|
done?: number | undefined
|
|
446
467
|
error?: string | undefined
|
|
447
468
|
}): IndexStatus {
|
|
469
|
+
// an empty error is none: the wire's field is a string
|
|
470
|
+
const error = status.error ? { error: status.error } : {}
|
|
448
471
|
switch (status.state) {
|
|
449
472
|
case 'rebuilding':
|
|
450
|
-
return { state: 'rebuilding', done: status.done ?? 0 }
|
|
473
|
+
return { state: 'rebuilding', done: status.done ?? 0, ...error }
|
|
451
474
|
case 'failed':
|
|
452
475
|
return { state: 'failed', error: status.error ?? '' }
|
|
453
476
|
default:
|
|
454
|
-
return { state: 'ready' }
|
|
477
|
+
return { state: 'ready', ...error }
|
|
455
478
|
}
|
|
456
479
|
}
|
|
457
480
|
|