@tinyshed/tinystore 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/backup.d.ts +2 -0
- package/dist/backup.js +1 -0
- package/dist/blobs.d.ts +18 -1
- package/dist/blobs.js +52 -45
- package/dist/console.d.ts +6 -3
- package/dist/console.js +53 -14
- package/dist/directory.d.ts +72 -0
- package/dist/directory.js +121 -0
- package/dist/index.d.ts +2 -1
- package/dist/jobs.d.ts +42 -14
- package/dist/jobs.js +146 -39
- package/dist/locks.js +18 -0
- package/dist/metrics.d.ts +5 -1
- package/dist/metrics.js +2 -0
- package/dist/query.d.ts +8 -1
- package/dist/query.js +56 -19
- package/dist/search.d.ts +17 -1
- package/dist/search.js +11 -2
- package/dist/sql.d.ts +33 -2
- package/dist/sql.js +91 -11
- package/dist/store.d.ts +9 -0
- package/dist/store.js +11 -0
- package/dist/wire/protocol.d.ts +219 -11
- package/dist/wire/protocol.js +650 -18
- package/package.json +8 -8
- package/src/backup.ts +3 -0
- package/src/blobs.ts +66 -45
- package/src/console.ts +56 -16
- package/src/directory.ts +235 -0
- package/src/index.ts +9 -0
- package/src/jobs.ts +187 -46
- package/src/locks.ts +22 -0
- package/src/metrics.ts +7 -1
- package/src/query.ts +68 -24
- package/src/search.ts +28 -5
- package/src/sql.ts +105 -11
- package/src/store.ts +12 -0
- package/src/wire/protocol.ts +685 -18
package/dist/jobs.js
CHANGED
|
@@ -5,14 +5,33 @@
|
|
|
5
5
|
// queue's jobs and hands each over on one stream, the answers going back on
|
|
6
6
|
// it. A handle opens with its first call on a connection.
|
|
7
7
|
import { FailureLog, messageOf } from './background.js';
|
|
8
|
-
import { CancelledError, CorruptError, InvalidError, UnavailableError } from './errors.js';
|
|
8
|
+
import { CancelledError, CorruptError, errorOf, InvalidError, UnavailableError } from './errors.js';
|
|
9
9
|
import { checkName, handleOn, viaLink } from './handles.js';
|
|
10
10
|
import { rateOf } from './limits.js';
|
|
11
11
|
import { sayOwn } from './logger.js';
|
|
12
12
|
import { check } from './schema.js';
|
|
13
13
|
import { ConnectionLostError } from './session.js';
|
|
14
14
|
import { ms, unixMs } from './time.js';
|
|
15
|
-
import { JobsAnswer, JobsCall, JobsChanged, JobsHeld, JobsId, JobsJob, JobsKept, JobsList, JobsPage, JobsQueueOpen, JobsScheduleOpen, JobsStep, JobsTx, JobsWork, methods, } from './wire/protocol.js';
|
|
15
|
+
import { Handle, JobsAnswer, JobsCall, JobsChanged, JobsCounts, JobsHeld, JobsId, JobsJob, JobsKept, JobsList, JobsPage, JobsQueueOpen, JobsScheduleOpen, JobsStep, JobsTx, JobsWork, methods, } from './wire/protocol.js';
|
|
16
|
+
/**
|
|
17
|
+
* Where a job without an id lies, which a cancel names it by: kept on the job
|
|
18
|
+
* a page gave, out of sight, and not on a copy of it.
|
|
19
|
+
*/
|
|
20
|
+
const ROW = Symbol('tinystore.job.row');
|
|
21
|
+
/** What a cancel sends of a job: its id, or for one without, its row. */
|
|
22
|
+
function namedBy(job, describe) {
|
|
23
|
+
if (typeof job === 'string') {
|
|
24
|
+
return { id: job };
|
|
25
|
+
}
|
|
26
|
+
if (job.id !== undefined) {
|
|
27
|
+
return { id: job.id };
|
|
28
|
+
}
|
|
29
|
+
const row = job[ROW];
|
|
30
|
+
if (row === undefined) {
|
|
31
|
+
throw new InvalidError(`${describe}: a job without an id names no row: cancel the job a list gave, not a copy`);
|
|
32
|
+
}
|
|
33
|
+
return { row };
|
|
34
|
+
}
|
|
16
35
|
/** The handlers a client's worker runs at once at most, as the server bounds them. */
|
|
17
36
|
const mostHandlers = 1024;
|
|
18
37
|
/** The pause before a worker dials again after a failure, doubling up to the last. */
|
|
@@ -247,10 +266,81 @@ async function getJob(source, id) {
|
|
|
247
266
|
const found = JobsJob.decode(body);
|
|
248
267
|
return found.found === true ? jobOf(source, found) : undefined;
|
|
249
268
|
}
|
|
250
|
-
async function cancelJob(source,
|
|
251
|
-
const
|
|
269
|
+
async function cancelJob(source, job) {
|
|
270
|
+
const named = namedBy(job, source.describe);
|
|
271
|
+
const body = await callOf(source, methods['jobs.cancel'], handle => JobsId.encode({ handle, ...named }), 'write');
|
|
252
272
|
return JobsChanged.decode(body).changed === true;
|
|
253
273
|
}
|
|
274
|
+
/**
|
|
275
|
+
* A page of the queue's jobs. A walk's page has a job whose value no longer
|
|
276
|
+
* reads as its error in its place; any other page fails on it.
|
|
277
|
+
*/
|
|
278
|
+
async function pageOf(source, options, walk) {
|
|
279
|
+
const body = await callOf(source, methods['jobs.list'], handle => JobsList.encode({ handle, ...options, walk }), 'read');
|
|
280
|
+
const page = JobsPage.decode(body);
|
|
281
|
+
const jobs = [];
|
|
282
|
+
for (const found of page.jobs ?? []) {
|
|
283
|
+
if (found.failure !== undefined) {
|
|
284
|
+
const { code, message, what } = found.failure;
|
|
285
|
+
jobs.push(errorOf(code ?? 'internal', message ?? '', what ?? {}));
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
const job = jobOf(source, found);
|
|
289
|
+
jobs.push(walk ? await job.catch((err) => err) : await job);
|
|
290
|
+
}
|
|
291
|
+
return { jobs, next: page.next };
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Every job a filter names, a page at a time. A job whose value no longer
|
|
295
|
+
* reads is its error in its place: `next()` rejects with it, and the call
|
|
296
|
+
* after goes on past it, as a `for await` that catches it and iterates the
|
|
297
|
+
* same walk again does.
|
|
298
|
+
*/
|
|
299
|
+
class Walk {
|
|
300
|
+
#source;
|
|
301
|
+
#options;
|
|
302
|
+
#jobs = [];
|
|
303
|
+
#after;
|
|
304
|
+
#ended = false;
|
|
305
|
+
constructor(source, options) {
|
|
306
|
+
this.#source = source;
|
|
307
|
+
this.#options = options;
|
|
308
|
+
}
|
|
309
|
+
async next() {
|
|
310
|
+
while (this.#jobs.length === 0 && !this.#ended) {
|
|
311
|
+
const page = await pageOf(this.#source, { ...this.#options, after: this.#after }, true);
|
|
312
|
+
this.#jobs = page.jobs;
|
|
313
|
+
this.#after = page.next;
|
|
314
|
+
this.#ended = page.next === undefined;
|
|
315
|
+
}
|
|
316
|
+
const job = this.#jobs.shift();
|
|
317
|
+
if (job === undefined) {
|
|
318
|
+
return { done: true, value: undefined };
|
|
319
|
+
}
|
|
320
|
+
if (job instanceof Error) {
|
|
321
|
+
throw job;
|
|
322
|
+
}
|
|
323
|
+
return { done: false, value: job };
|
|
324
|
+
}
|
|
325
|
+
async return() {
|
|
326
|
+
this.#end();
|
|
327
|
+
return { done: true, value: undefined };
|
|
328
|
+
}
|
|
329
|
+
async throw(err) {
|
|
330
|
+
this.#end();
|
|
331
|
+
throw err;
|
|
332
|
+
}
|
|
333
|
+
[Symbol.asyncIterator]() {
|
|
334
|
+
return this;
|
|
335
|
+
}
|
|
336
|
+
async [Symbol.asyncDispose]() {
|
|
337
|
+
this.#end();
|
|
338
|
+
}
|
|
339
|
+
#end() {
|
|
340
|
+
this.#jobs = [];
|
|
341
|
+
this.#ended = true;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
254
344
|
/** A job as the server sent it, its value read as the queue reads one. */
|
|
255
345
|
async function jobOf(source, found) {
|
|
256
346
|
let value;
|
|
@@ -258,9 +348,10 @@ async function jobOf(source, found) {
|
|
|
258
348
|
value = await source.values.decode(found.value ?? 'null');
|
|
259
349
|
}
|
|
260
350
|
catch (err) {
|
|
261
|
-
|
|
351
|
+
const named = found.id === undefined ? 'a job without an id' : `id ${JSON.stringify(found.id)}`;
|
|
352
|
+
throw new CorruptError(`${source.describe}: ${named}: ${messageOf(err)}`);
|
|
262
353
|
}
|
|
263
|
-
|
|
354
|
+
const job = {
|
|
264
355
|
id: found.id,
|
|
265
356
|
value,
|
|
266
357
|
state: (found.state ?? 'waiting'),
|
|
@@ -278,6 +369,10 @@ async function jobOf(source, found) {
|
|
|
278
369
|
endedAt: new Date(found.endedAt ?? found.startedAt),
|
|
279
370
|
},
|
|
280
371
|
};
|
|
372
|
+
if (found.row !== undefined) {
|
|
373
|
+
Object.defineProperty(job, ROW, { value: found.row });
|
|
374
|
+
}
|
|
375
|
+
return job;
|
|
281
376
|
}
|
|
282
377
|
/**
|
|
283
378
|
* Follows the job under id over the server's watch, until the stream's last
|
|
@@ -384,21 +479,38 @@ export class Queue {
|
|
|
384
479
|
await callOf(this.#source, methods['jobs.set'], handle => JobsCall.encode({ ...fields, handle }), 'write');
|
|
385
480
|
}
|
|
386
481
|
/**
|
|
387
|
-
* Changes the id's job when it
|
|
388
|
-
*
|
|
389
|
-
* job that runs,
|
|
482
|
+
* Changes the id's job when it is not running: its value, the steps kept
|
|
483
|
+
* for the old one dropped with it, and its time, group or repeat when the
|
|
484
|
+
* options give them. It says whether it did: a job that runs, is done or
|
|
485
|
+
* failed, or never was answers false.
|
|
390
486
|
*/
|
|
391
487
|
async update(id, value, options = {}) {
|
|
392
488
|
const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) };
|
|
393
489
|
return (await this.#changed(methods['jobs.update'], fields)) === true;
|
|
394
490
|
}
|
|
395
491
|
/**
|
|
396
|
-
* Takes
|
|
397
|
-
*
|
|
492
|
+
* Takes a job, whatever its state, and says whether it was there: the job
|
|
493
|
+
* under an id, or a job a list gave, by its id or, for one without, by the
|
|
494
|
+
* row it was listed from. A job retried, run again for its repeat or
|
|
495
|
+
* cancelled since is in no row, and its cancel takes nothing, never another
|
|
496
|
+
* job. A running one's handler sees its signal abort, and what it returns
|
|
398
497
|
* then settles nothing; a repeating one stops.
|
|
498
|
+
*
|
|
499
|
+
* for await (const job of emails.all({ state: 'scheduled' })) if (job.value.to === bounced) await emails.cancel(job)
|
|
399
500
|
*/
|
|
400
|
-
cancel(
|
|
401
|
-
return cancelJob(this.#source,
|
|
501
|
+
cancel(job) {
|
|
502
|
+
return cancelJob(this.#source, job);
|
|
503
|
+
}
|
|
504
|
+
/** How many jobs the queue holds in each state, from one snapshot; it reads no value. */
|
|
505
|
+
async counts() {
|
|
506
|
+
const body = await callOf(this.#source, methods['jobs.counts'], handle => Handle.encode({ handle }), 'read');
|
|
507
|
+
const { scheduled, waiting, running, failed } = JobsCounts.decode(body);
|
|
508
|
+
return {
|
|
509
|
+
scheduled: scheduled ?? 0,
|
|
510
|
+
waiting: waiting ?? 0,
|
|
511
|
+
running: running ?? 0,
|
|
512
|
+
failed: failed ?? 0,
|
|
513
|
+
};
|
|
402
514
|
}
|
|
403
515
|
/**
|
|
404
516
|
* Where the job under id is: scheduled, waiting and how many jobs are
|
|
@@ -420,27 +532,21 @@ export class Queue {
|
|
|
420
532
|
watch(id) {
|
|
421
533
|
return watchJob(this.#source, id);
|
|
422
534
|
}
|
|
423
|
-
/**
|
|
535
|
+
/**
|
|
536
|
+
* A page of the queue's jobs: with no prefix every job, those with ids by id,
|
|
537
|
+
* then those without as they run, then those without that failed.
|
|
538
|
+
*/
|
|
424
539
|
async list(options = {}) {
|
|
425
|
-
const
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
let after;
|
|
436
|
-
for (;;) {
|
|
437
|
-
const page = await this.list({ ...options, after });
|
|
438
|
-
yield* page.jobs;
|
|
439
|
-
if (page.next === undefined) {
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
after = page.next;
|
|
443
|
-
}
|
|
540
|
+
const page = await pageOf(this.#source, options, false);
|
|
541
|
+
return { jobs: page.jobs, next: page.next };
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Every job the options name, a page at a time, holding nothing between
|
|
545
|
+
* pages. A job whose value no longer reads is a CorruptError in its place,
|
|
546
|
+
* naming it, and iterating the same walk again goes on past it.
|
|
547
|
+
*/
|
|
548
|
+
all(options = {}) {
|
|
549
|
+
return new Walk(this.#source, options);
|
|
444
550
|
}
|
|
445
551
|
/**
|
|
446
552
|
* Runs handler on each job as it falls due, as many at once as the
|
|
@@ -491,8 +597,8 @@ export class TxQueue {
|
|
|
491
597
|
update(id, value, options = {}) {
|
|
492
598
|
return this.#queue.update(id, value, options);
|
|
493
599
|
}
|
|
494
|
-
cancel(
|
|
495
|
-
return this.#queue.cancel(
|
|
600
|
+
cancel(job) {
|
|
601
|
+
return this.#queue.cancel(job);
|
|
496
602
|
}
|
|
497
603
|
/** The job under id as the transaction sees it, its own writes among it. */
|
|
498
604
|
get(id) {
|
|
@@ -555,12 +661,13 @@ export class StoreTxQueue {
|
|
|
555
661
|
async set(id, value, options = {}) {
|
|
556
662
|
this.#write('jobs.set', { ...jobFields(options), id }, value);
|
|
557
663
|
}
|
|
558
|
-
/** Changes the id's job when it
|
|
664
|
+
/** Changes the id's job when it is not running, its old value's steps dropped, with the commit. */
|
|
559
665
|
async update(id, value, options = {}) {
|
|
560
666
|
this.#write('jobs.update', { ...jobFields(options), id }, value);
|
|
561
667
|
}
|
|
562
|
-
/** Takes the job under id, whatever its state, with the commit. */
|
|
563
|
-
async cancel(
|
|
668
|
+
/** Takes the job under id, or a job a list gave, whatever its state, with the commit. */
|
|
669
|
+
async cancel(job) {
|
|
670
|
+
const id = namedBy(job, this.#source.describe);
|
|
564
671
|
this.#keep({ source: this.#source, method: methods['jobs.cancel'], id });
|
|
565
672
|
}
|
|
566
673
|
#write(method, fields, value) {
|
|
@@ -577,7 +684,7 @@ export async function commitJobs(link, ops) {
|
|
|
577
684
|
writes.push({
|
|
578
685
|
method: op.method,
|
|
579
686
|
call: op.call === undefined ? undefined : { ...op.call, handle },
|
|
580
|
-
id: op.id === undefined ? undefined : { handle,
|
|
687
|
+
id: op.id === undefined ? undefined : { handle, ...op.id },
|
|
581
688
|
});
|
|
582
689
|
}
|
|
583
690
|
await connection.session.call(methods['jobs.tx'], JobsTx.encode({ writes }));
|
package/dist/locks.js
CHANGED
|
@@ -52,6 +52,7 @@ export class Semaphore {
|
|
|
52
52
|
if (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 1000) {
|
|
53
53
|
throw new InvalidError(`semaphore ${name}: a limit of ${options.limit}, not 1 to 1000`);
|
|
54
54
|
}
|
|
55
|
+
keepLimit(link, name, options.limit);
|
|
55
56
|
this.name = name;
|
|
56
57
|
this.limit = options.limit;
|
|
57
58
|
this.#keys = new Keys(link, 'semaphore', name, options.limit, options);
|
|
@@ -63,6 +64,23 @@ export class Semaphore {
|
|
|
63
64
|
return this.#keys.hold(key, fn, 0, permitOf);
|
|
64
65
|
}
|
|
65
66
|
}
|
|
67
|
+
// The limit each semaphore's first open gave, by the link of the store that
|
|
68
|
+
// opened it: one name keeps one limit in a program, and two stores are two
|
|
69
|
+
// programs, each of whose holds the store admits by its own limit.
|
|
70
|
+
const opened = new WeakMap();
|
|
71
|
+
/** Keeps the limit a semaphore's first open on this store gives: another limit is refused, naming both. */
|
|
72
|
+
function keepLimit(link, name, limit) {
|
|
73
|
+
let limits = opened.get(link);
|
|
74
|
+
if (limits === undefined) {
|
|
75
|
+
limits = new Map();
|
|
76
|
+
opened.set(link, limits);
|
|
77
|
+
}
|
|
78
|
+
const first = limits.get(name) ?? limit;
|
|
79
|
+
if (first !== limit) {
|
|
80
|
+
throw new InvalidError(`semaphore ${name}: open in this program with a limit of ${first}, not ${limit}`);
|
|
81
|
+
}
|
|
82
|
+
limits.set(name, first);
|
|
83
|
+
}
|
|
66
84
|
function waitOf(options) {
|
|
67
85
|
return ms(options?.wait ?? defaultWait);
|
|
68
86
|
}
|
package/dist/metrics.d.ts
CHANGED
|
@@ -96,6 +96,8 @@ export interface AggregateQuery<O extends Op = Op> extends Query {
|
|
|
96
96
|
without?: readonly string[] | undefined;
|
|
97
97
|
/** a timer's percentile also gives a point between its bounds, interpolated: an estimate */
|
|
98
98
|
estimate?: boolean | undefined;
|
|
99
|
+
/** a NaN sample counts in no bucket, which says how many it passed over in skipped; an infinity is still refused */
|
|
100
|
+
skipNaN?: boolean | undefined;
|
|
99
101
|
}
|
|
100
102
|
/** What every bucket of an aggregate says of itself. */
|
|
101
103
|
interface Spanned {
|
|
@@ -104,6 +106,8 @@ interface Spanned {
|
|
|
104
106
|
/** how many samples it comes from; a timer's, how many durations */
|
|
105
107
|
samples: number;
|
|
106
108
|
resets: number;
|
|
109
|
+
/** the NaN samples it passed over, with skipNaN */
|
|
110
|
+
skipped: number;
|
|
107
111
|
/** its value comes from less than its span: the range, the clock or retention cut it */
|
|
108
112
|
partial: boolean;
|
|
109
113
|
/** its first step began before the bucket before it: a whole bucket between had no samples */
|
|
@@ -131,7 +135,7 @@ export interface Aggregate<O extends Op = Op> {
|
|
|
131
135
|
name: string;
|
|
132
136
|
/** the labels by kept or without left */
|
|
133
137
|
labels: Labels;
|
|
134
|
-
/** a bucket without samples is left out */
|
|
138
|
+
/** a bucket without samples is left out, and so is one of NaN alone */
|
|
135
139
|
buckets: Bucket<O>[];
|
|
136
140
|
/** a page cut the group here: the page its next reads begins with the rest of its buckets */
|
|
137
141
|
more: boolean;
|
package/dist/metrics.js
CHANGED
|
@@ -367,6 +367,7 @@ function aggregateOf(query) {
|
|
|
367
367
|
by: query.by === undefined ? undefined : [...query.by],
|
|
368
368
|
without: query.without === undefined ? undefined : [...query.without],
|
|
369
369
|
estimate: query.estimate === true,
|
|
370
|
+
skipNan: query.skipNaN === true,
|
|
370
371
|
};
|
|
371
372
|
}
|
|
372
373
|
function pageOf(options) {
|
|
@@ -377,6 +378,7 @@ function bucketOf(bucket) {
|
|
|
377
378
|
at: new Date(bucket.at ?? 0),
|
|
378
379
|
samples: bucket.samples ?? 0,
|
|
379
380
|
resets: bucket.resets ?? 0,
|
|
381
|
+
skipped: bucket.skipped ?? 0,
|
|
380
382
|
partial: bucket.partial === true,
|
|
381
383
|
gap: bucket.gap === true,
|
|
382
384
|
};
|
package/dist/query.d.ts
CHANGED
|
@@ -123,7 +123,14 @@ export declare class Query<T = Row> implements Embeddable {
|
|
|
123
123
|
scalar<V = SqlValue>(): Promise<V>;
|
|
124
124
|
/** Every row the conditions match, without the order and the limit. */
|
|
125
125
|
count(): Promise<number>;
|
|
126
|
-
/**
|
|
126
|
+
/**
|
|
127
|
+
* The rows one at a time, as their parts arrive: for more than fits in
|
|
128
|
+
* memory. The server reads them from one snapshot of five seconds at
|
|
129
|
+
* most; ordered by columns by their names, a unique one last, as `list`
|
|
130
|
+
* is, it reads on past that after the last row it sent, and any other
|
|
131
|
+
* walk fails `limit` there. Inside a transaction it reads the rows `all`
|
|
132
|
+
* does.
|
|
133
|
+
*/
|
|
127
134
|
each(): AsyncGenerator<T>;
|
|
128
135
|
/**
|
|
129
136
|
* A page after the last row of the one before, by the query's order,
|
package/dist/query.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
var _a;
|
|
6
6
|
import { CorruptError, InvalidError } from './errors.js';
|
|
7
7
|
import { check } from './schema.js';
|
|
8
|
-
import { cellOf, conditionOf, grouped, identOf, joinedWith, placeholders, Sql, } from './sql.js';
|
|
8
|
+
import { cellOf, checked, conditionOf, grouped, identOf, joinedWith, placeholders, Sql, walk, } from './sql.js';
|
|
9
9
|
/**
|
|
10
10
|
* A query on a table: a value, each call returning a new one, so that the
|
|
11
11
|
* first stays as it was. `all`, `one`, `scalar`, `count`, `each` and `list`
|
|
@@ -36,7 +36,7 @@ export class Query {
|
|
|
36
36
|
}
|
|
37
37
|
/** The columns a row holds, as SQL text, and the shape of the row they make. */
|
|
38
38
|
select(columns) {
|
|
39
|
-
const select = typeof columns === 'string' ? new Sql(columns, []) : columns;
|
|
39
|
+
const select = checked(typeof columns === 'string' ? new Sql(columns, []) : columns);
|
|
40
40
|
return new _a(this.#source, { ...this.#parts, select });
|
|
41
41
|
}
|
|
42
42
|
/**
|
|
@@ -102,7 +102,7 @@ export class Query {
|
|
|
102
102
|
throw new InvalidError(`an order is asc or desc, not ${JSON.stringify(direction)}`);
|
|
103
103
|
}
|
|
104
104
|
const order = column instanceof Sql
|
|
105
|
-
? { piece: column, direction }
|
|
105
|
+
? { piece: checked(column), direction }
|
|
106
106
|
: { piece: new Sql(identOf(column), []), column, direction };
|
|
107
107
|
return this.#with({ order: [...this.#parts.order, order] });
|
|
108
108
|
}
|
|
@@ -150,14 +150,21 @@ export class Query {
|
|
|
150
150
|
const answered = await this.#source.runner.rows(statement, 'scalar');
|
|
151
151
|
return Number(answered.rows[0]?.[0] ?? 0);
|
|
152
152
|
}
|
|
153
|
-
/**
|
|
153
|
+
/**
|
|
154
|
+
* The rows one at a time, as their parts arrive: for more than fits in
|
|
155
|
+
* memory. The server reads them from one snapshot of five seconds at
|
|
156
|
+
* most; ordered by columns by their names, a unique one last, as `list`
|
|
157
|
+
* is, it reads on past that after the last row it sent, and any other
|
|
158
|
+
* walk fails `limit` there. Inside a transaction it reads the rows `all`
|
|
159
|
+
* does.
|
|
160
|
+
*/
|
|
154
161
|
async *each() {
|
|
155
162
|
const runner = this.#source.runner;
|
|
156
|
-
if (runner
|
|
163
|
+
if (runner[walk] === undefined) {
|
|
157
164
|
yield* await this.all();
|
|
158
165
|
return;
|
|
159
166
|
}
|
|
160
|
-
for await (const row of runner
|
|
167
|
+
for await (const row of runner[walk](render(this.#parts), walkOrder(this.#parts.order))) {
|
|
161
168
|
yield (await this.#rows(Object.keys(row), [Object.values(row)]))[0];
|
|
162
169
|
}
|
|
163
170
|
}
|
|
@@ -174,9 +181,14 @@ export class Query {
|
|
|
174
181
|
if (order.length === 0 || order.some(each => each.column === undefined)) {
|
|
175
182
|
throw new InvalidError('a page is read by the query order, of columns by their names: orderBy(…) first');
|
|
176
183
|
}
|
|
184
|
+
// A page reads its limit's rows after its cursor: the query's own limit or
|
|
185
|
+
// offset would be dropped, and limit(100) page through every row.
|
|
186
|
+
if (this.#parts.limit !== undefined || this.#parts.offset !== undefined) {
|
|
187
|
+
throw new InvalidError('list after a limit or an offset: list pages the rows itself and takes its own limit');
|
|
188
|
+
}
|
|
177
189
|
const limit = whole(options.limit ?? 100, 'a page limit');
|
|
178
|
-
const digest = digestOf(render(
|
|
179
|
-
let parts = { ...this.#parts, limit
|
|
190
|
+
const digest = digestOf(render(this.#parts));
|
|
191
|
+
let parts = { ...this.#parts, limit };
|
|
180
192
|
if (options.after !== undefined) {
|
|
181
193
|
const after = cursorOf(options.after, digest);
|
|
182
194
|
parts = {
|
|
@@ -219,8 +231,9 @@ export class Query {
|
|
|
219
231
|
for (const name of names) {
|
|
220
232
|
const value = values[name];
|
|
221
233
|
if (value instanceof Sql) {
|
|
222
|
-
|
|
223
|
-
|
|
234
|
+
const set = checked(value);
|
|
235
|
+
sets.push(`${identOf(name)} = ${set.text}`);
|
|
236
|
+
out.push(...set.values);
|
|
224
237
|
}
|
|
225
238
|
else {
|
|
226
239
|
sets.push(`${identOf(name)} = ?`);
|
|
@@ -272,7 +285,7 @@ export class Query {
|
|
|
272
285
|
return new _a(this.#source, { ...this.#parts, ...changed });
|
|
273
286
|
}
|
|
274
287
|
#joined(kind, table, on) {
|
|
275
|
-
const condition = typeof on === 'string' ? new Sql(on, []) : on;
|
|
288
|
+
const condition = checked(typeof on === 'string' ? new Sql(on, []) : on);
|
|
276
289
|
const ref = tableRef(table);
|
|
277
290
|
const join = new Sql(`${kind} ${ref.text} on ${condition.text}`, condition.values);
|
|
278
291
|
return this.#with({ joins: [...this.#parts.joins, join] });
|
|
@@ -286,6 +299,19 @@ export class Query {
|
|
|
286
299
|
if (this.#parts.joins.length > 0) {
|
|
287
300
|
throw new InvalidError(`${what} of a join: write it in SQL`);
|
|
288
301
|
}
|
|
302
|
+
// SQLite's update and delete take none of these: limit(10).delete()
|
|
303
|
+
// would delete every row the conditions match.
|
|
304
|
+
const parts = this.#parts;
|
|
305
|
+
const shaped = parts.limit !== undefined ||
|
|
306
|
+
parts.offset !== undefined ||
|
|
307
|
+
parts.order.length > 0 ||
|
|
308
|
+
parts.groups.length > 0 ||
|
|
309
|
+
parts.having.length > 0 ||
|
|
310
|
+
parts.select !== undefined ||
|
|
311
|
+
parts.includes.length > 0;
|
|
312
|
+
if (shaped) {
|
|
313
|
+
throw new InvalidError(`${what} with a limit, offset, order, group, select or include: narrow it by conditions, or write it in SQL`);
|
|
314
|
+
}
|
|
289
315
|
}
|
|
290
316
|
}
|
|
291
317
|
_a = Query;
|
|
@@ -338,7 +364,7 @@ export class Table extends Query {
|
|
|
338
364
|
const values = [...insert.values];
|
|
339
365
|
if (options.where !== undefined) {
|
|
340
366
|
// in an upsert's update, a column is the row's that holds the key
|
|
341
|
-
const where = conditionOf(options.where);
|
|
367
|
+
const where = checked(conditionOf(options.where));
|
|
342
368
|
text += ` where ${where.text}`;
|
|
343
369
|
values.push(...where.values);
|
|
344
370
|
}
|
|
@@ -605,18 +631,15 @@ function kept(condition, values) {
|
|
|
605
631
|
if (counted.positional !== values.length || counted.named.length > 0 || counted.numbered) {
|
|
606
632
|
throw new InvalidError(`${counted.positional} placeholders and ${values.length} values: each ? takes one value`);
|
|
607
633
|
}
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
piece: new Sql(condition, values.map((value, at) => cellOf(value, `value ${at + 1}`))),
|
|
611
|
-
bare: false,
|
|
612
|
-
},
|
|
613
|
-
];
|
|
634
|
+
const piece = new Sql(condition, values.map((value, at) => cellOf(value, `value ${at + 1}`)));
|
|
635
|
+
return [{ piece: checked(piece), bare: false }];
|
|
614
636
|
}
|
|
615
637
|
if (values.length > 0) {
|
|
616
638
|
throw new InvalidError('a condition that is not text carries its values: pass it alone');
|
|
617
639
|
}
|
|
618
640
|
if (condition instanceof Sql) {
|
|
619
|
-
|
|
641
|
+
const piece = checked(condition);
|
|
642
|
+
return [{ piece, bare: piece.bare || grouped(piece.text) }];
|
|
620
643
|
}
|
|
621
644
|
return Object.keys(condition)
|
|
622
645
|
.sort()
|
|
@@ -676,6 +699,20 @@ function render(parts) {
|
|
|
676
699
|
}
|
|
677
700
|
return new Sql(text, values);
|
|
678
701
|
}
|
|
702
|
+
/**
|
|
703
|
+
* The columns a walk's rows are ordered by, as its rows name them, when each
|
|
704
|
+
* is a column by its name: the server reads on past one snapshot by them, as
|
|
705
|
+
* `list` pages by them.
|
|
706
|
+
*/
|
|
707
|
+
function walkOrder(order) {
|
|
708
|
+
if (order.length === 0 || order.some(each => each.column === undefined)) {
|
|
709
|
+
return undefined;
|
|
710
|
+
}
|
|
711
|
+
return order.map(each => ({
|
|
712
|
+
column: each.column.split('.').pop(),
|
|
713
|
+
descending: each.direction === 'desc',
|
|
714
|
+
}));
|
|
715
|
+
}
|
|
679
716
|
/**
|
|
680
717
|
* The condition that starts a page after a row's order values. SQLite orders
|
|
681
718
|
* a null before every value, first in ascending order and last in descending,
|
package/dist/search.d.ts
CHANGED
|
@@ -62,6 +62,13 @@ export interface Query {
|
|
|
62
62
|
vectorWeight?: number | undefined;
|
|
63
63
|
rerank?: boolean | undefined;
|
|
64
64
|
} | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* the least cosine, from -1 to 1, a document's vector has with the
|
|
67
|
+
* search's for the vector to find it: a search by its vector alone leaves
|
|
68
|
+
* out the hits under it, and one by words too adds nothing for them to
|
|
69
|
+
* the sum; invalid beside a rerank
|
|
70
|
+
*/
|
|
71
|
+
minSimilarity?: number | undefined;
|
|
65
72
|
/**
|
|
66
73
|
* the last word read as the start of a word too, as a search box types:
|
|
67
74
|
* `crypt` finds `cryptography` as well, at half its field's weight; a
|
|
@@ -103,18 +110,27 @@ export interface Hit<T> {
|
|
|
103
110
|
score: number;
|
|
104
111
|
/** where the search's words matched it, when the search asked with `matches` */
|
|
105
112
|
matches?: Match[];
|
|
113
|
+
/** the score its words gave it, as a search by its words alone scores it, where they found it among their best */
|
|
114
|
+
textScore?: number;
|
|
115
|
+
/** the cosine of its vector with the search's, from -1 to 1, where both have one */
|
|
116
|
+
similarity?: number;
|
|
106
117
|
}
|
|
107
118
|
/** What a search found, the best first, and the text it read in fact when a word was misspelt. */
|
|
108
119
|
export interface Found<T> {
|
|
109
120
|
hits: Hit<T>[];
|
|
110
121
|
corrected?: string | undefined;
|
|
111
122
|
}
|
|
112
|
-
/**
|
|
123
|
+
/**
|
|
124
|
+
* Where an index stands: serving what it was opened with, read again beside it, or a rebuild that failed.
|
|
125
|
+
* `error` beside `ready` and `rebuilding` is the last maintenance pass's failure while the passes keep failing.
|
|
126
|
+
*/
|
|
113
127
|
export type IndexStatus = {
|
|
114
128
|
state: 'ready';
|
|
129
|
+
error?: string | undefined;
|
|
115
130
|
} | {
|
|
116
131
|
state: 'rebuilding';
|
|
117
132
|
done: number;
|
|
133
|
+
error?: string | undefined;
|
|
118
134
|
} | {
|
|
119
135
|
state: 'failed';
|
|
120
136
|
error: string;
|
package/dist/search.js
CHANGED
|
@@ -48,6 +48,7 @@ class Searchable {
|
|
|
48
48
|
vector,
|
|
49
49
|
vectorWeight: asked.hybrid?.vectorWeight,
|
|
50
50
|
rerank: asked.hybrid?.rerank,
|
|
51
|
+
minSimilarity: asked.minSimilarity,
|
|
51
52
|
prefix: asked.prefix === true || undefined,
|
|
52
53
|
// none unless asked, so that a server before it reads the search
|
|
53
54
|
prefixEvery: asked.prefix === 'every' || undefined,
|
|
@@ -67,6 +68,12 @@ class Searchable {
|
|
|
67
68
|
if (hit.matches !== undefined) {
|
|
68
69
|
read.matches = hit.matches.map(matchOf);
|
|
69
70
|
}
|
|
71
|
+
if (hit.textScore !== undefined) {
|
|
72
|
+
read.textScore = hit.textScore;
|
|
73
|
+
}
|
|
74
|
+
if (hit.similarity !== undefined) {
|
|
75
|
+
read.similarity = hit.similarity;
|
|
76
|
+
}
|
|
70
77
|
return read;
|
|
71
78
|
});
|
|
72
79
|
return found.corrected === undefined ? { hits } : { hits, corrected: found.corrected };
|
|
@@ -213,13 +220,15 @@ export async function listIndexes(link) {
|
|
|
213
220
|
});
|
|
214
221
|
}
|
|
215
222
|
function statusOf(status) {
|
|
223
|
+
// an empty error is none: the wire's field is a string
|
|
224
|
+
const error = status.error ? { error: status.error } : {};
|
|
216
225
|
switch (status.state) {
|
|
217
226
|
case 'rebuilding':
|
|
218
|
-
return { state: 'rebuilding', done: status.done ?? 0 };
|
|
227
|
+
return { state: 'rebuilding', done: status.done ?? 0, ...error };
|
|
219
228
|
case 'failed':
|
|
220
229
|
return { state: 'failed', error: status.error ?? '' };
|
|
221
230
|
default:
|
|
222
|
-
return { state: 'ready' };
|
|
231
|
+
return { state: 'ready', ...error };
|
|
223
232
|
}
|
|
224
233
|
}
|
|
225
234
|
function matchOf(found) {
|