@jarenjs/db 0.46.5 → 0.49.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +106 -2
- package/README.md +136 -2
- package/dist/types/algebra.d.ts +34 -3
- package/dist/types/dialect.d.ts +5 -0
- package/dist/types/errors.d.ts +3 -0
- package/dist/types/live-time.d.ts +141 -0
- package/dist/types/live.d.ts +3 -1
- package/dist/types/plan.d.ts +2 -0
- package/dist/types/query.d.ts +2 -1
- package/dist/types/residual.d.ts +5 -2
- package/dist/types/series.d.ts +227 -0
- package/dist/types/store.d.ts +8 -1
- package/docs/LIVE-FORMAT.md +103 -0
- package/docs/MODEL-FORMAT.md +19 -0
- package/package.json +4 -4
- package/src/algebra.js +22 -3
- package/src/dialect.js +13 -0
- package/src/dialects/sqlite.js +19 -0
- package/src/emit.js +32 -6
- package/src/errors.js +3 -0
- package/src/live-time.js +596 -0
- package/src/live.js +41 -8
- package/src/plan.js +706 -16
- package/src/query.js +160 -11
- package/src/residual.js +15 -6
- package/src/series.js +349 -0
- package/src/store.js +22 -3
- package/types/index.d.ts +54 -2
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The temporal recognizer: which documents ask a §8.16 question,
|
|
3
|
+
* which of those a declared `(series, at)` index can answer, and what
|
|
4
|
+
* the honest reason is when it cannot.
|
|
5
|
+
*
|
|
6
|
+
* No SQL and no storage kind live here. The physical feature is the
|
|
7
|
+
* composite JSONPath index a model already declares
|
|
8
|
+
* (`{ "name": "by_series_at", "path": ["$.series", "$.at"] }`); this
|
|
9
|
+
* module only decides which of the three CLOSED shapes a planned
|
|
10
|
+
* selection is in:
|
|
11
|
+
*
|
|
12
|
+
* 1. **range** — an equality on every leading column of an instant
|
|
13
|
+
* index plus a half-open range on the instant column, ordered by
|
|
14
|
+
* the instant. The index seeks; nothing is left over.
|
|
15
|
+
* 2. **as-of** — the same prefix with ONE instant bound, ordered by
|
|
16
|
+
* the instant and cut to a finite window. One index seek per probe.
|
|
17
|
+
* 3. **bucket** — a fixed-width ladder over the instant column with
|
|
18
|
+
* the exact `sum|mean|min|max|count` aggregates, which is a
|
|
19
|
+
* `GROUP BY` over integer arithmetic.
|
|
20
|
+
*
|
|
21
|
+
* Everything else — a calendar ladder, a fill policy, a rolling window,
|
|
22
|
+
* an as-of JOIN, `first`/`last` — is a named core refinement: the
|
|
23
|
+
* database narrows through the index and `@jarenjs/core/series` (via
|
|
24
|
+
* the residual, which is the ENGINE running the caller's own document)
|
|
25
|
+
* decides. The narrowing is the contribution; the answer is always the
|
|
26
|
+
* engine's, which is what makes a refinement idempotent.
|
|
27
|
+
*
|
|
28
|
+
* Every refusal here has a CODE, and the code is the first word of the
|
|
29
|
+
* sentence the plan carries, so `explain().series.reasons[].code` and
|
|
30
|
+
* `explain().residual.reasons[].reason` cannot drift apart.
|
|
31
|
+
*/
|
|
32
|
+
/** The three §8.16 operators a whole document can BE. */
|
|
33
|
+
export declare const SERIES_ROOT_OPS: readonly string[];
|
|
34
|
+
/** Every §8.16 operator: naming one makes a document temporal. */
|
|
35
|
+
export declare const SERIES_OPS: readonly string[];
|
|
36
|
+
/**
|
|
37
|
+
* The D5 aggregates a `GROUP BY` reproduces exactly, and the plan's
|
|
38
|
+
* name for each. `count` is `rows` because it counts SOURCE ROWS —
|
|
39
|
+
* duplicates and measured gaps included — which is `COUNT(*)` and not
|
|
40
|
+
* `COUNT(value)`; the six value aggregates skip a `null` reading,
|
|
41
|
+
* which is what SQL's aggregates already do with SQL `NULL`.
|
|
42
|
+
*
|
|
43
|
+
* `first` and `last` are deliberately absent: they name a row by its
|
|
44
|
+
* position in the series, and a group's order is not the series' order.
|
|
45
|
+
*/
|
|
46
|
+
export declare const NATIVE_AGGREGATES: Readonly<{
|
|
47
|
+
mean: "avg";
|
|
48
|
+
sum: "sum";
|
|
49
|
+
min: "min";
|
|
50
|
+
max: "max";
|
|
51
|
+
count: "rows";
|
|
52
|
+
}>;
|
|
53
|
+
/**
|
|
54
|
+
* The closed reason table. A reason is a CODE and a sentence; the plan
|
|
55
|
+
* carries `"<code>: <sentence>"` so one string serves strict mode's
|
|
56
|
+
* refusal, `explain().residual.reasons` and the machine-readable
|
|
57
|
+
* `explain().series.reasons[].code` at once.
|
|
58
|
+
*/
|
|
59
|
+
export declare const SERIES_REASONS: Readonly<{
|
|
60
|
+
'missing-series-prefix': string;
|
|
61
|
+
'calendar-width': string;
|
|
62
|
+
'named-zone': string;
|
|
63
|
+
'fill-policy': string;
|
|
64
|
+
'rolling-refinement': string;
|
|
65
|
+
'asof-refinement': "an as-of join walks both sides once, so the index bounds the fetch and the kernel joins";
|
|
66
|
+
'nonliteral-spec': string;
|
|
67
|
+
'unsupported-aggregate': string;
|
|
68
|
+
'row-selector': string;
|
|
69
|
+
'instant-not-integer': string;
|
|
70
|
+
'value-not-numeric': string;
|
|
71
|
+
'nonnative-grouping': "the grouping key or the projection is not the closed bucket shape";
|
|
72
|
+
'invalid-spec': string;
|
|
73
|
+
}>;
|
|
74
|
+
/**
|
|
75
|
+
* One reason, in both spellings at once.
|
|
76
|
+
* @param {keyof SERIES_REASONS | string} code
|
|
77
|
+
* @param {string} construct - the operator or clause that forced it
|
|
78
|
+
* @returns {{ code: string, construct: string, reason: string }}
|
|
79
|
+
*/
|
|
80
|
+
export declare function seriesReason(code: keyof SERIES_REASONS | string, construct: string): {
|
|
81
|
+
code: string;
|
|
82
|
+
construct: string;
|
|
83
|
+
reason: string;
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Every declared index whose LAST covered column is `column`, with the
|
|
87
|
+
* columns before it as the prefix that must be pinned.
|
|
88
|
+
*
|
|
89
|
+
* `minColumns` is what keeps an ordinary query ordinary. A collection
|
|
90
|
+
* that declares `(age)` and is asked for `age > 21` is not asking a
|
|
91
|
+
* temporal question, and nothing in a column can say otherwise — so
|
|
92
|
+
* the shape D9 actually names, a COMPOSITE index whose last column is
|
|
93
|
+
* the instant, is what makes a plain selection temporal. A document
|
|
94
|
+
* that named a §8.16 operator has already said so itself, and reads
|
|
95
|
+
* the singular index too.
|
|
96
|
+
* @param {any} shape - { indexes?: { name, columns }[] }
|
|
97
|
+
* @param {string} column
|
|
98
|
+
* @param {number} [minColumns]
|
|
99
|
+
* @returns {{ name: string, prefix: string[], column: string }[]}
|
|
100
|
+
*/
|
|
101
|
+
export declare function instantIndexesOver(shape: any, column: string, minColumns?: number): {
|
|
102
|
+
name: string;
|
|
103
|
+
prefix: string[];
|
|
104
|
+
column: string;
|
|
105
|
+
}[];
|
|
106
|
+
/**
|
|
107
|
+
* The index a fetch actually SEEKS through, or `null` when none does.
|
|
108
|
+
*
|
|
109
|
+
* A B-tree is seekable exactly as far as its leading columns are
|
|
110
|
+
* decided: a run of equalities, and then at most one range. So the
|
|
111
|
+
* index that wins is the one with the longest leading run of PINNED
|
|
112
|
+
* columns whose next column is the instant the query ranges over —
|
|
113
|
+
* which is `(series, at)` under an equality on the series, and is
|
|
114
|
+
* nothing at all under a bare instant bound, because a range on a
|
|
115
|
+
* trailing column reads every row of the index.
|
|
116
|
+
*
|
|
117
|
+
* With no instant column of its own (an as-of join reading an instant
|
|
118
|
+
* the model does not index) a pinned prefix alone still seeks, and is
|
|
119
|
+
* reported as what it is.
|
|
120
|
+
* @param {any} shape
|
|
121
|
+
* @param {string | null} column - the instant column, or `null`
|
|
122
|
+
* @param {{ pinned: Set<string>, bounds: Map<string, any> }} facts
|
|
123
|
+
* @param {number} [minColumns] - see {@link instantIndexesOver}
|
|
124
|
+
* @returns {{ name: string, prefix: string[], column: string | null } | null}
|
|
125
|
+
*/
|
|
126
|
+
export declare function seekingIndexFor(shape: any, column: string | null, facts: {
|
|
127
|
+
pinned: Set<string>;
|
|
128
|
+
bounds: Map<string, any>;
|
|
129
|
+
}, minColumns?: number): {
|
|
130
|
+
name: string;
|
|
131
|
+
prefix: string[];
|
|
132
|
+
column: string | null;
|
|
133
|
+
} | null;
|
|
134
|
+
/**
|
|
135
|
+
* Walk a pushed filter and report, per column, what it decided: which
|
|
136
|
+
* columns an equality pinned and what instant bounds a range put on
|
|
137
|
+
* one. Only a top-level conjunction counts — a disjunction or a
|
|
138
|
+
* negation decides nothing about a seek.
|
|
139
|
+
* @param {import('./algebra.js').PlanPredicate | null} filter
|
|
140
|
+
* @returns {{ pinned: Set<string>,
|
|
141
|
+
* bounds: Map<string, { from: any, fromOp: string | null,
|
|
142
|
+
* to: any, toOp: string | null }> }}
|
|
143
|
+
*/
|
|
144
|
+
export declare function filterFacts(filter: import('./algebra.js').PlanPredicate | null): {
|
|
145
|
+
pinned: Set<string>;
|
|
146
|
+
bounds: Map<string, {
|
|
147
|
+
from: any;
|
|
148
|
+
fromOp: string | null;
|
|
149
|
+
to: any;
|
|
150
|
+
toOp: string | null;
|
|
151
|
+
}>;
|
|
152
|
+
};
|
|
153
|
+
/**
|
|
154
|
+
* The fixed ladder a `$time-bucket`/`$resample` spec asks for, or the
|
|
155
|
+
* reason it is not one. `origin` is folded to an epoch here — a
|
|
156
|
+
* `{ offset }` context moves the ladder's default anchor off UTC's
|
|
157
|
+
* midnight, which is arithmetic, while a named zone is not.
|
|
158
|
+
*
|
|
159
|
+
* The width is read through the temporal kernel's OWN compiler, so
|
|
160
|
+
* `'PT1H'`, `3600000` and `'PT60M'` are the same ladder, the default
|
|
161
|
+
* anchor is the kernel's rather than a second guess at it, and a width
|
|
162
|
+
* mixing the two families was already refused when the query compiled.
|
|
163
|
+
* @param {{ every: any, origin?: any, zone?: any, offset?: any }} spec
|
|
164
|
+
* @param {(spec: any, options: any) => any} compileBuckets - the kernel's
|
|
165
|
+
* @returns {{ every: number, origin: number } | { code: string }}
|
|
166
|
+
*/
|
|
167
|
+
export declare function fixedLadder(spec: {
|
|
168
|
+
every: any;
|
|
169
|
+
origin?: any;
|
|
170
|
+
zone?: any;
|
|
171
|
+
offset?: any;
|
|
172
|
+
}, compileBuckets: (spec: any, options: any) => any): {
|
|
173
|
+
every: number;
|
|
174
|
+
origin: number;
|
|
175
|
+
} | {
|
|
176
|
+
code: string;
|
|
177
|
+
};
|
|
178
|
+
/**
|
|
179
|
+
* Whether a `PlanRef` can carry a native bucket ladder: the instant
|
|
180
|
+
* must be a declared whole epoch, because the boundary arithmetic in
|
|
181
|
+
* SQL is integer arithmetic and a truncating division over a real
|
|
182
|
+
* would put an instant before 1970 in the bucket after its own.
|
|
183
|
+
* @param {import('./algebra.js').PlanRef | null} ref
|
|
184
|
+
* @returns {string | null} the reason code, or `null` when it can
|
|
185
|
+
*/
|
|
186
|
+
export declare function instantRefusal(ref: import('./algebra.js').PlanRef | null): string | null;
|
|
187
|
+
/**
|
|
188
|
+
* Whether a `PlanRef` can carry a native VALUE aggregate.
|
|
189
|
+
* @param {import('./algebra.js').PlanRef | null} ref
|
|
190
|
+
* @returns {string | null}
|
|
191
|
+
*/
|
|
192
|
+
export declare function valueRefusal(ref: import('./algebra.js').PlanRef | null): string | null;
|
|
193
|
+
/**
|
|
194
|
+
* The one member name a `'$.on'`-style row selector reads, or `null`
|
|
195
|
+
* for anything a declared column cannot stand in for. The language's
|
|
196
|
+
* own reader (`compileSelector`) folds a single-segment path to a bare
|
|
197
|
+
* name; this reads the same two spellings out of the FROZEN literal a
|
|
198
|
+
* planner sees, and refuses everything else rather than guessing.
|
|
199
|
+
* @param {any} text
|
|
200
|
+
* @returns {string | null}
|
|
201
|
+
*/
|
|
202
|
+
export declare function singularSelector(text: any): string | null;
|
|
203
|
+
/**
|
|
204
|
+
* The explain record for one temporal document. Counts are the LAST
|
|
205
|
+
* ACTUAL execution's — never an estimate — and are `null` until the
|
|
206
|
+
* document has run once.
|
|
207
|
+
* @param {{ mode: 'native' | 'hybrid' | 'engine', operation: string,
|
|
208
|
+
* index?: string | null, prefix?: string[], range?: any,
|
|
209
|
+
* ladder?: any, aggregates?: string[], refinement?: string | null,
|
|
210
|
+
* reasons?: { code: string, construct: string, reason: string }[] }} facts
|
|
211
|
+
* @returns {any}
|
|
212
|
+
*/
|
|
213
|
+
export declare function seriesRecord(facts: {
|
|
214
|
+
mode: 'native' | 'hybrid' | 'engine';
|
|
215
|
+
operation: string;
|
|
216
|
+
index?: string | null;
|
|
217
|
+
prefix?: string[];
|
|
218
|
+
range?: any;
|
|
219
|
+
ladder?: any;
|
|
220
|
+
aggregates?: string[];
|
|
221
|
+
refinement?: string | null;
|
|
222
|
+
reasons?: {
|
|
223
|
+
code: string;
|
|
224
|
+
construct: string;
|
|
225
|
+
reason: string;
|
|
226
|
+
}[];
|
|
227
|
+
}): any;
|
package/dist/types/store.d.ts
CHANGED
|
@@ -33,8 +33,14 @@ export declare function normalizeModel(model: any): Map<string, any>;
|
|
|
33
33
|
* @param {{ driver: any, path?: string, compileSchema?: Function,
|
|
34
34
|
* busyTimeout?: number, queueTimeout?: number, journalMode?: string,
|
|
35
35
|
* statementCacheBound?: number, profile?: any, operators?: any,
|
|
36
|
-
* functions?: any, extensions?: any,
|
|
36
|
+
* functions?: any, extensions?: any, zoneProvider?: any,
|
|
37
37
|
* readOnly?: boolean }} options
|
|
38
|
+
* `zoneProvider` is D7's injected clock: a named zone in a temporal
|
|
39
|
+
* spec (`{ "every": "P1M", "zone": "Europe/Amsterdam" }`) is host code
|
|
40
|
+
* the database cannot have, so a store that never received one refuses
|
|
41
|
+
* such a document (`JQ0003`) rather than answering it in UTC. It
|
|
42
|
+
* reaches every residual compilation, which is where the calendar
|
|
43
|
+
* ladder actually walks.
|
|
38
44
|
* @returns {Promise<any>}
|
|
39
45
|
*/
|
|
40
46
|
export declare function openStore(model: any, options: {
|
|
@@ -49,5 +55,6 @@ export declare function openStore(model: any, options: {
|
|
|
49
55
|
operators?: any;
|
|
50
56
|
functions?: any;
|
|
51
57
|
extensions?: any;
|
|
58
|
+
zoneProvider?: any;
|
|
52
59
|
readOnly?: boolean;
|
|
53
60
|
}): Promise<any>;
|
package/docs/LIVE-FORMAT.md
CHANGED
|
@@ -148,6 +148,7 @@ against its own result document (§9). Registration:
|
|
|
148
148
|
const live = await store.collection('users').live(document, {
|
|
149
149
|
externals: {}, // fixed at registration (§8)
|
|
150
150
|
mode: 'auto', // 'auto' | 'incremental' | 'rerun'
|
|
151
|
+
eventTime: undefined, // a temporal view's watermark (§13)
|
|
151
152
|
});
|
|
152
153
|
live.result; // the maintained result document
|
|
153
154
|
live.mode; // { strategy, mode: 'incremental'|'rerun', reason }
|
|
@@ -177,6 +178,8 @@ what the pushdown planner already means by it.
|
|
|
177
178
|
| `orderBy` beside a refined spatial predicate — over `$distance` (not a path) or over a member (the set residual drops the planner's order terms) | **re-run on invalidation**, the ordering named as the reason | the previous result, for diffing |
|
|
178
179
|
| a whole-query aggregate or a `groupBy` whose `where` is a refined spatial predicate | **re-run on invalidation** — the accumulator needs a fully translated selection and a refinement is not one; the reason says so | the previous result, for diffing |
|
|
179
180
|
| a spatial predicate the planner **refused** (no `derive` index on the member, an untyped member, an unbounded probe) | **re-run on invalidation**, the refusal named — it never translated, so nothing narrows the fetch | the previous result, for diffing |
|
|
181
|
+
| a `$resample` or `$rolling` document over the collection, with an explicit `eventTime` and a fixed width (§13) | **event-time bucket / rolling state**: rows kept by bucket, or in instant order; only what a write can reach is folded again, through `@jarenjs/core/series` itself | the contributing rows, plus one fold per bucket |
|
|
182
|
+
| the same document with no `eventTime`, a calendar width, a named zone, a `locf`/`linear` fill, a `first`/`last` aggregate, or a retention that does not cover the window | **re-run on invalidation**, the member that stopped it named (§13.2) | the previous result, for diffing |
|
|
180
183
|
| joins, multi-entity roots, graph loads, every entity query | **re-run on invalidation — declared, not attempted** in this version | the previous result, for diffing |
|
|
181
184
|
| anything else: non-translatable predicates, `limit` without `orderBy`, `offset` > 0, windowed aggregates, `@jarenjs/linq`'s nested two-level `groupBy` emission, non-canonical group returns | **re-run on invalidation**, the reason named | the previous result, for diffing |
|
|
182
185
|
|
|
@@ -376,3 +379,103 @@ the signal), no maintenance over asynchronous connections in this
|
|
|
376
379
|
version (every current driver is synchronous; the browser driver's
|
|
377
380
|
order owns that story), no replication, and no ordering guarantee for
|
|
378
381
|
unordered queries beyond §9's determinism.
|
|
382
|
+
|
|
383
|
+
## 13. Event time
|
|
384
|
+
|
|
385
|
+
A live view over time needs to know what "now" is — which reading counts
|
|
386
|
+
as late — and the machine's clock is a different quantity from the
|
|
387
|
+
instant a reading carries. So there is no clock in this layer and none
|
|
388
|
+
under it: the **watermark arrives**.
|
|
389
|
+
|
|
390
|
+
```js
|
|
391
|
+
const live = await store.collection('readings').live(
|
|
392
|
+
[{ $resample: ['$[*]', { every: 60_000, aggregate: 'mean' }] }],
|
|
393
|
+
{ eventTime: {
|
|
394
|
+
path: '$.at', // the instant member, as a row selector
|
|
395
|
+
watermark: 1767225600000, // a finite epoch the HOST supplies
|
|
396
|
+
allowedLateness: 300_000, // how late a reading may still be
|
|
397
|
+
retention: 900_000, // the horizon this view claims
|
|
398
|
+
} });
|
|
399
|
+
|
|
400
|
+
live.advance(1767225660000); // the only way a watermark moves
|
|
401
|
+
live.stats().watermark; // what it is now
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
`eventTime` is a **closed** member set: `path`, `watermark`,
|
|
405
|
+
`allowedLateness` (default 0) and `retention`. Anything else — a
|
|
406
|
+
misspelling, a non-finite epoch, a negative lateness, a `path` that is
|
|
407
|
+
not a singular row selector — is `JD0053` at registration, not a member
|
|
408
|
+
quietly ignored. `advance()` refuses a value that is not finite or that
|
|
409
|
+
goes backwards (a `TypeError`), and it is absent on every view
|
|
410
|
+
registered without an `eventTime`. An entity document has no collection
|
|
411
|
+
to place rows in and re-runs, so an `eventTime` on `store.live` is
|
|
412
|
+
`JD0053` too.
|
|
413
|
+
|
|
414
|
+
### 13.1 What is maintained
|
|
415
|
+
|
|
416
|
+
Two documents, and only these two shapes: `$resample` and `$rolling`
|
|
417
|
+
whose series operand is the collection (`"$[*]"`, or a FLWOR over it
|
|
418
|
+
whose `$where` narrows and whose `$return` is the bare binding).
|
|
419
|
+
|
|
420
|
+
- **A bucket view keeps its rows by bucket.** A write touches one bucket
|
|
421
|
+
— two, when it moves a reading across a boundary — and exactly those
|
|
422
|
+
are folded again by calling `resampleSeries` over that bucket's own
|
|
423
|
+
rows. The aggregate is therefore the kernel's, and cannot drift from
|
|
424
|
+
what a fresh query would answer.
|
|
425
|
+
- **A rolling view keeps its rows in instant order.** A write at `t` can
|
|
426
|
+
only change the windows ending in `[t, t + width)`, so exactly that
|
|
427
|
+
stretch is recomputed — again by the kernel, over the slice those
|
|
428
|
+
windows can see.
|
|
429
|
+
|
|
430
|
+
`retention` is the horizon the view claims, and it is checked rather
|
|
431
|
+
than assumed: it must cover the width plus `allowedLateness`, which is
|
|
432
|
+
the span a single repair can read. A shorter one is a re-run with the
|
|
433
|
+
numbers printed. It is **not** a compaction policy — the maintained
|
|
434
|
+
state is bounded by `live.maxMaintained` exactly as every other
|
|
435
|
+
strategy's is (§12), and nothing an answer still depends on is dropped.
|
|
436
|
+
That is the honest statement of what this version buys: bounded repair
|
|
437
|
+
work and a visible lateness contract, not a smaller heap.
|
|
438
|
+
|
|
439
|
+
### 13.2 What re-runs, and why
|
|
440
|
+
|
|
441
|
+
| Refused | Because |
|
|
442
|
+
|---|---|
|
|
443
|
+
| no `eventTime` | a temporal view maintains event time, and a hidden clock is the one source this suite will not use |
|
|
444
|
+
| a calendar `every`/`width` (`P1M`, `P1D` on a zone) | a calendar ladder walks a wall clock and a month has no width, so its boundaries move with the data rather than with arithmetic |
|
|
445
|
+
| a named `zone` | it resolves through the injected provider, which maintenance would have to consult per boundary |
|
|
446
|
+
| `fill: 'locf'` / `'linear'` | they fill an empty bucket from its neighbours, so one late reading moves buckets it never belonged to |
|
|
447
|
+
| `aggregate: 'first'` / `'last'` | they name a row by its position in the series, which a per-key state does not preserve — the same refusal the pushdown planner makes |
|
|
448
|
+
| a `retention` under `width + allowedLateness` | a repair could read outside the horizon the view claims |
|
|
449
|
+
| a `$subsequence` window, a projecting operand, a collection with no document key | there is no row a key can be tracked through |
|
|
450
|
+
| a spec whose `at` selector is not `eventTime.path` | the state would place a row by one instant and aggregate it by another |
|
|
451
|
+
|
|
452
|
+
Each of those is `live.mode.reason`, and `mode: 'incremental'` still
|
|
453
|
+
refuses them at registration with `JD0051`.
|
|
454
|
+
|
|
455
|
+
### 13.3 Late data is visible, never lost
|
|
456
|
+
|
|
457
|
+
A reading is **late** when its instant — before the write, after it, or
|
|
458
|
+
both — is behind `watermark - allowedLateness`. Late readings are not
|
|
459
|
+
dropped and not quietly folded in. The view **re-reads** from the store,
|
|
460
|
+
so the answer still equals what a fresh query would give, and the
|
|
461
|
+
emission carries the reason:
|
|
462
|
+
|
|
463
|
+
```js
|
|
464
|
+
live.subscribe(({ patch, seq, lateData }) => {
|
|
465
|
+
if (lateData !== undefined) {
|
|
466
|
+
// { reason: 'late-data', at, key, watermark, allowedLateness, boundary }
|
|
467
|
+
}
|
|
468
|
+
});
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
`stats().lateData` counts them and `stats().reruns` counts the re-reads
|
|
472
|
+
they forced. A reading outside the view's own `start`/`end` window is
|
|
473
|
+
not late data: it belongs to no bucket this view maintains, so there is
|
|
474
|
+
nothing to be late for.
|
|
475
|
+
|
|
476
|
+
The maintained answer is **equal to a full recomputation after every
|
|
477
|
+
mutation** — not approximately, and not eventually. `test/db/live-time.test.js`
|
|
478
|
+
holds a shuffled stream of inserts, in-place updates, instant moves and
|
|
479
|
+
deletes against `resampleSeries` / `rollingSeries` over the whole
|
|
480
|
+
collection after each one, which is the only oracle that cannot drift
|
|
481
|
+
with the implementation.
|
package/docs/MODEL-FORMAT.md
CHANGED
|
@@ -71,6 +71,24 @@ The store runs over SQLite — on Node (`@jarenjs/db/node`), on Bun
|
|
|
71
71
|
An invalid model document is `JD0005` with a `docPath` pointing at the
|
|
72
72
|
offending member. Model checking happens before any database work.
|
|
73
73
|
|
|
74
|
+
**A time series needs no new declaration.** A composite index whose
|
|
75
|
+
LAST path is a finite numeric epoch member is the whole physical
|
|
76
|
+
feature a temporal plan reads:
|
|
77
|
+
|
|
78
|
+
```json
|
|
79
|
+
{ "indexes": [{ "name": "by_series_at", "path": ["$.series", "$.at"] }] }
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
There is no `derive` kind for it, no column type and no host function.
|
|
83
|
+
The columns before the instant are the prefix an equality has to pin
|
|
84
|
+
for the index to seek, exactly as for any other composite index, and
|
|
85
|
+
the schema is the type source as always: type the instant `integer` and
|
|
86
|
+
a bucket ladder is pushed as integer arithmetic; type it `number` and
|
|
87
|
+
the ladder stays a core refinement, because a truncating division over
|
|
88
|
+
a real would put an instant in the wrong bucket. Which shapes are
|
|
89
|
+
pushed, and the reason each refinement is one, are in ARCHITECTURE.md's
|
|
90
|
+
"The temporal plan".
|
|
91
|
+
|
|
74
92
|
### 2.1 Derived indexes (spatial and vector storage)
|
|
75
93
|
|
|
76
94
|
A generated column must be a scalar (§3), and a GeoJSON position is an
|
|
@@ -713,6 +731,7 @@ error.
|
|
|
713
731
|
| `JD0050` | live queries require change capture |
|
|
714
732
|
| `JD0051` | the demanded live mode is unavailable |
|
|
715
733
|
| `JD0052` | the live-query bound was reached |
|
|
734
|
+
| `JD0053` | the live event-time declaration is invalid |
|
|
716
735
|
| `JD2001` | insert found the key already present |
|
|
717
736
|
| `JD2002` | a usable key could not be resolved for the write |
|
|
718
737
|
| `JD2003` | the write failed schema validation |
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jarenjs/db",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.49.2",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
7
7
|
"types": "./types/index.d.ts",
|
|
@@ -71,9 +71,9 @@
|
|
|
71
71
|
"prepack": "npm run build:types"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@jarenjs/core": "^0.
|
|
75
|
-
"@jarenjs/json": "^0.
|
|
76
|
-
"@jarenjs/validate": "^0.
|
|
74
|
+
"@jarenjs/core": "^0.49.2",
|
|
75
|
+
"@jarenjs/json": "^0.49.2",
|
|
76
|
+
"@jarenjs/validate": "^0.49.2"
|
|
77
77
|
},
|
|
78
78
|
"bin": {
|
|
79
79
|
"jaren-db": "./src/cli.js"
|
package/src/algebra.js
CHANGED
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
* One plan shape covers this version: a guarded selection over ONE
|
|
12
12
|
* collection with optional ordering, window, aggregate and a
|
|
13
13
|
* whole-document projection — or, instead of an ordering and a window,
|
|
14
|
-
* a k-nearest RANK the engine finishes over the rows the plan fetches
|
|
15
|
-
*
|
|
16
|
-
*
|
|
14
|
+
* a k-nearest RANK the engine finishes over the rows the plan fetches,
|
|
15
|
+
* or, instead of a projection, a fixed-width temporal BUCKET the plan
|
|
16
|
+
* groups and aggregates itself. Constructs beyond it are residuals by
|
|
17
|
+
* design (see ARCHITECTURE.md's deliberate-residual table).
|
|
17
18
|
*/
|
|
18
19
|
|
|
19
20
|
/** The plan format version, carried on every plan. */
|
|
@@ -57,6 +58,21 @@ export const PLAN_VERSION = 2;
|
|
|
57
58
|
*
|
|
58
59
|
* @typedef {{ ref: PlanRef, desc: boolean, emptyGreatest: boolean }} PlanOrderTerm
|
|
59
60
|
*
|
|
61
|
+
* @typedef {{ ref: PlanRef, every: number, origin: number, as: string,
|
|
62
|
+
* order: 'asc' | 'desc' | 'first-seen',
|
|
63
|
+
* aggregates: { fn: 'rows' | 'sum' | 'avg' | 'min' | 'max',
|
|
64
|
+
* ref: PlanRef | null, as: string,
|
|
65
|
+
* empty: 'null' | 'zero' | 'omit' }[] }} PlanBucket
|
|
66
|
+
* The fixed-width temporal GROUP BY: the instant column, the ladder's
|
|
67
|
+
* width and anchor in epoch milliseconds, the name the bucket's start
|
|
68
|
+
* is answered under, how the groups are ordered, and one aggregate
|
|
69
|
+
* per answered member. `rows` is `COUNT(*)` — the D5 count of SOURCE
|
|
70
|
+
* rows, duplicates and measured gaps included — and the four value
|
|
71
|
+
* aggregates skip a `NULL` reading exactly as the kernel skips a
|
|
72
|
+
* `null` one. `first-seen` order is the group's earliest row identity,
|
|
73
|
+
* which is the engine's own "order of first appearance" (§6.5).
|
|
74
|
+
* A plan carrying a bucket carries no `aggregate` and no `rank`.
|
|
75
|
+
*
|
|
60
76
|
* @typedef {{ column: string, dims: number,
|
|
61
77
|
* probe: { lit: number[] } | { ext: string },
|
|
62
78
|
* offset: number, limit: number, margin: number }} PlanRank
|
|
@@ -79,6 +95,7 @@ export const PLAN_VERSION = 2;
|
|
|
79
95
|
* order: PlanOrderTerm[] | null,
|
|
80
96
|
* window: { offset: number, limit: number | null } | null,
|
|
81
97
|
* rank: PlanRank | null,
|
|
98
|
+
* bucket: PlanBucket | null,
|
|
82
99
|
* aggregate: { fn: 'count' | 'sum' | 'avg' | 'min' | 'max',
|
|
83
100
|
* ref: PlanRef | null } | null,
|
|
84
101
|
* project: 'document',
|
|
@@ -99,6 +116,7 @@ export function selectPlan(collection) {
|
|
|
99
116
|
order: null,
|
|
100
117
|
window: null,
|
|
101
118
|
rank: null,
|
|
119
|
+
bucket: null,
|
|
102
120
|
aggregate: null,
|
|
103
121
|
project: 'document',
|
|
104
122
|
};
|
|
@@ -123,6 +141,7 @@ export function conjoin(filter, predicate) {
|
|
|
123
141
|
const SQL_TOKENS = [
|
|
124
142
|
'SELECT', 'WHERE', 'ORDER BY', 'LIMIT ', 'INSERT', 'FROM ',
|
|
125
143
|
'jsonb_extract', 'json_type', 'substr(', 'instr(', '"doc"', '@p1', ' AS ',
|
|
144
|
+
'GROUP BY', 'COUNT(',
|
|
126
145
|
];
|
|
127
146
|
|
|
128
147
|
/**
|
package/src/dialect.js
CHANGED
|
@@ -58,6 +58,9 @@
|
|
|
58
58
|
* strEndsWith: (valueSql: string, patternA: string, patternB: string, patternC: string) => string,
|
|
59
59
|
* strContains: (valueSql: string, patternSql: string) => string,
|
|
60
60
|
* orderNulls: (nullsFirst: boolean) => string,
|
|
61
|
+
* timeBucket: (instantSql: string, originSql: string, everyA: string,
|
|
62
|
+
* everyB: string, everyC: string) => string,
|
|
63
|
+
* groupAggregate: (fn: string, valueSql: string | null) => string,
|
|
61
64
|
* rowIdentity: () => string,
|
|
62
65
|
* identityIn: (identitySql: string, paramSqls: string[]) => string,
|
|
63
66
|
* rtree?: { module: string, columns: readonly string[] },
|
|
@@ -473,6 +476,16 @@ export function createDialect(spec) {
|
|
|
473
476
|
strEndsWith: spec.strEndsWith,
|
|
474
477
|
strContains: spec.strContains,
|
|
475
478
|
orderNulls: spec.orderNulls,
|
|
479
|
+
/**
|
|
480
|
+
* The instant a fixed-width bucket ladder labels one row with:
|
|
481
|
+
* `origin + floor((at - origin) / every) * every`, which reduces to
|
|
482
|
+
* `at` less the non-negative remainder. The parameters appear in
|
|
483
|
+
* TEXT order — the origin once, the width three times — because a
|
|
484
|
+
* positional dialect numbers them by where they are written.
|
|
485
|
+
*/
|
|
486
|
+
timeBucket: spec.timeBucket,
|
|
487
|
+
/** One grouped aggregate; `null` counts ROWS rather than values. */
|
|
488
|
+
groupAggregate: spec.groupAggregate,
|
|
476
489
|
rowIdentity: spec.rowIdentity,
|
|
477
490
|
/**
|
|
478
491
|
* The R\*Tree spelling: the module name and the virtual table's own
|
package/src/dialects/sqlite.js
CHANGED
|
@@ -82,6 +82,9 @@ export const sqliteDialect = createDialect({
|
|
|
82
82
|
upsert: true,
|
|
83
83
|
savepoints: true,
|
|
84
84
|
alterTableFull: false,
|
|
85
|
+
// a GROUP BY / ORDER BY term may name a result alias, so a bucket
|
|
86
|
+
// ladder is written once rather than three times
|
|
87
|
+
groupByAlias: true,
|
|
85
88
|
},
|
|
86
89
|
tableSuffix: ' STRICT',
|
|
87
90
|
// RFC 3339 text → epoch milliseconds, in SQL: the migration planner
|
|
@@ -152,6 +155,22 @@ export const sqliteDialect = createDialect({
|
|
|
152
155
|
`(length(${patternA}) = 0 OR substr(${valueSql}, -length(${patternB})) = ${patternC})`,
|
|
153
156
|
strContains: (valueSql, patternSql) => `instr(${valueSql}, ${patternSql}) > 0`,
|
|
154
157
|
orderNulls: (nullsFirst) => (nullsFirst ? ' NULLS FIRST' : ' NULLS LAST'),
|
|
158
|
+
// the fixed bucket ladder, in integer arithmetic all the way down.
|
|
159
|
+
// `origin + floor((at - origin) / every) * every` is `at` less the
|
|
160
|
+
// NON-NEGATIVE remainder, and `((x % m) + m) % m` is how a language
|
|
161
|
+
// whose `%` truncates towards zero (C's, and SQLite's) spells one —
|
|
162
|
+
// which is the whole of why an instant before 1970 lands in its own
|
|
163
|
+
// bucket rather than the one after it. The column is declared
|
|
164
|
+
// INTEGER, so nothing here converts and nothing rounds.
|
|
165
|
+
timeBucket: (instantSql, originSql, everyA, everyB, everyC) =>
|
|
166
|
+
`(${instantSql} - (((${instantSql} - ${originSql}) % ${everyA} + ${everyB}) % ${everyC}))`,
|
|
167
|
+
// `rows` is COUNT(*) — the D5 count of SOURCE rows, which is not
|
|
168
|
+
// COUNT(value): a measured gap is a row that reported nothing, and
|
|
169
|
+
// the difference between "nobody reported" and "everybody reported a
|
|
170
|
+
// gap" is exactly what the count is for
|
|
171
|
+
groupAggregate: (fn, valueSql) => (valueSql === null
|
|
172
|
+
? 'COUNT(*)'
|
|
173
|
+
: `${{ sum: 'SUM', avg: 'AVG', min: 'MIN', max: 'MAX' }[fn]}(${valueSql})`),
|
|
155
174
|
rowIdentity: () => '"rowid"',
|
|
156
175
|
// membership of the row identity in a bound list — the fetch of a
|
|
157
176
|
// k-nearest plan's candidates. `IN` over the rowid is a primary-key
|
package/src/emit.js
CHANGED
|
@@ -273,6 +273,17 @@ export function emitPlan(plan, dialect, physical) {
|
|
|
273
273
|
}
|
|
274
274
|
};
|
|
275
275
|
|
|
276
|
+
// The temporal bucket's ladder, written ONCE and named: the SELECT
|
|
277
|
+
// list carries it, and the grouping and the ordering name the alias.
|
|
278
|
+
// Writing it three times would triple its parameters, and a bucket
|
|
279
|
+
// start is exactly the kind of value the caller wants to read back.
|
|
280
|
+
const bucketSql = plan.bucket === null ? null
|
|
281
|
+
: dialect.timeBucket(valueOf(plan.bucket.ref),
|
|
282
|
+
param({ literal: plan.bucket.origin }),
|
|
283
|
+
param({ literal: plan.bucket.every }),
|
|
284
|
+
param({ literal: plan.bucket.every }),
|
|
285
|
+
param({ literal: plan.bucket.every }));
|
|
286
|
+
|
|
276
287
|
const selection = plan.rank !== null
|
|
277
288
|
// the k-nearest fetch: the row identity and the packed column
|
|
278
289
|
// under the pushed WHERE, and nothing that orders or limits — the
|
|
@@ -280,15 +291,30 @@ export function emitPlan(plan, dialect, physical) {
|
|
|
280
291
|
// the rank loses to fetching the column and ranking in the engine,
|
|
281
292
|
// and none of them runs where no function can be registered)
|
|
282
293
|
? `${dialect.rowIdentity()} AS ${q('rid')}, ${q(plan.rank.column)} AS ${q('vec')}`
|
|
283
|
-
: plan.
|
|
284
|
-
? `${
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
294
|
+
: plan.bucket !== null
|
|
295
|
+
? [`${bucketSql} AS ${q(plan.bucket.as)}`,
|
|
296
|
+
...plan.bucket.aggregates.map((entry) =>
|
|
297
|
+
`${dialect.groupAggregate(entry.fn,
|
|
298
|
+
entry.ref === null ? null : valueOf(entry.ref))} AS ${q(entry.as)}`)].join(', ')
|
|
299
|
+
: plan.aggregate === null
|
|
300
|
+
? `${dialect.jsonText(docColumn)} AS ${q('doc')}`
|
|
301
|
+
: plan.aggregate.fn === 'count'
|
|
302
|
+
? `COUNT(*) AS ${q('value')}`
|
|
303
|
+
: `${plan.aggregate.fn.toUpperCase()}(${valueOf(plan.aggregate.ref)}) AS ${q('value')}`;
|
|
288
304
|
|
|
289
305
|
let sql = `SELECT ${selection} FROM ${q(physical.table)}`;
|
|
290
306
|
if (plan.filter !== null) sql += ` WHERE ${emitPred(plan.filter)}`;
|
|
291
|
-
if (plan.
|
|
307
|
+
if (plan.bucket !== null) {
|
|
308
|
+
// `first-seen` is the engine's own group order (§6.5, first
|
|
309
|
+
// appearance), which over a collection is the group's earliest row
|
|
310
|
+
// identity — the same tiebreaker the ungrouped fetch appends
|
|
311
|
+
const alias = q(plan.bucket.as);
|
|
312
|
+
const order = plan.bucket.order === 'first-seen'
|
|
313
|
+
? dialect.groupAggregate('min', dialect.rowIdentity())
|
|
314
|
+
: `${alias} ${plan.bucket.order === 'desc' ? 'DESC' : 'ASC'}`;
|
|
315
|
+
sql += ` GROUP BY ${alias} ORDER BY ${order}`;
|
|
316
|
+
}
|
|
317
|
+
if (plan.aggregate === null && plan.rank === null && plan.bucket === null) {
|
|
292
318
|
const terms = (plan.order ?? []).map((term) => {
|
|
293
319
|
// Jaren's default sorts an empty key least: NULLS FIRST when
|
|
294
320
|
// ascending, NULLS LAST when descending — and mirrored for
|
package/src/errors.js
CHANGED
|
@@ -34,6 +34,7 @@ export const DB_CODES = Object.freeze({
|
|
|
34
34
|
JD0050: 'live queries require change capture',
|
|
35
35
|
JD0051: 'the demanded live mode is unavailable',
|
|
36
36
|
JD0052: 'the live-query bound was reached',
|
|
37
|
+
JD0053: 'the live event-time declaration is invalid',
|
|
37
38
|
JD0020: "the migration's from-shape does not match the database",
|
|
38
39
|
JD0021: 'the migration is missing a required data transform',
|
|
39
40
|
JD0022: 'an applied migration disagrees with the history record',
|
|
@@ -94,6 +95,8 @@ export const DB_CODES = Object.freeze({
|
|
|
94
95
|
* classifies as re-run; the reason names the forcing construct
|
|
95
96
|
* - `JD0052` — registering would exceed the store's `live.maxQueries`
|
|
96
97
|
* bound; the bound is printed, never silent
|
|
98
|
+
* - `JD0053` — a live query's `eventTime` names a member it does not
|
|
99
|
+
* admit, or a watermark/retention that is not a finite span
|
|
97
100
|
* - `JD0020` — a migration's `from` hash does not match the
|
|
98
101
|
* database's recorded shape; running it would corrupt
|
|
99
102
|
* - `JD0021` — a draft transform was not filled in, or a document no
|