@jarenjs/db 0.46.5 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +133 -17
- package/README.md +270 -36
- package/docs/JOBS-FORMAT.md +24 -8
- package/docs/LIVE-FORMAT.md +139 -7
- package/docs/MIGRATION-FORMAT.md +118 -36
- package/docs/MODEL-FORMAT.md +251 -30
- package/package.json +4 -5
- package/schemas/jaren-migration.draft-07.schema.json +73 -0
- package/schemas/jaren-migration.schema.json +73 -0
- package/src/algebra.js +22 -3
- package/src/capture.js +66 -28
- package/src/cli.js +225 -44
- package/src/ddl.js +23 -3
- package/src/dialect.js +13 -0
- package/src/dialects/sqlite.js +21 -1
- package/src/driver.js +63 -16
- package/src/drivers/wasm.js +1 -0
- package/src/emit-model.js +14 -0
- package/src/emit.js +42 -9
- package/src/entity.js +92 -47
- package/src/errors.js +28 -0
- package/src/index.js +2 -2
- package/src/jobs.js +40 -5
- package/src/live-time.js +605 -0
- package/src/live.js +52 -9
- package/src/migrate.js +397 -191
- package/src/model.js +173 -8
- package/src/plan.js +834 -47
- package/src/query.js +296 -22
- package/src/residual.js +15 -6
- package/src/series.js +349 -0
- package/src/store.js +243 -69
- package/src/tracker.js +173 -48
- package/types/index.d.ts +206 -12
- package/types/node.d.ts +3 -1
- package/types/typed.d.ts +58 -2
- package/types/wasm.d.ts +7 -0
- package/dist/types/algebra.d.ts +0 -199
- package/dist/types/app.d.ts +0 -49
- package/dist/types/capture.d.ts +0 -85
- package/dist/types/cli.d.ts +0 -2
- package/dist/types/dag-job.d.ts +0 -40
- package/dist/types/ddl.d.ts +0 -229
- package/dist/types/derive.d.ts +0 -250
- package/dist/types/dialect.d.ts +0 -149
- package/dist/types/dialects/sqlite.d.ts +0 -9
- package/dist/types/driver.d.ts +0 -110
- package/dist/types/drivers/bun.d.ts +0 -47
- package/dist/types/drivers/node.d.ts +0 -37
- package/dist/types/drivers/wasm.d.ts +0 -65
- package/dist/types/emit-model.d.ts +0 -44
- package/dist/types/emit.d.ts +0 -75
- package/dist/types/entity.d.ts +0 -23
- package/dist/types/errors.d.ts +0 -167
- package/dist/types/graph.d.ts +0 -28
- package/dist/types/index.d.ts +0 -37
- package/dist/types/jobs.d.ts +0 -140
- package/dist/types/knn.d.ts +0 -69
- package/dist/types/live.d.ts +0 -62
- package/dist/types/migrate.d.ts +0 -170
- package/dist/types/model.d.ts +0 -36
- package/dist/types/patch-sql.d.ts +0 -37
- package/dist/types/plan.d.ts +0 -140
- package/dist/types/profile.d.ts +0 -80
- package/dist/types/query.d.ts +0 -111
- package/dist/types/residual.d.ts +0 -61
- package/dist/types/store.d.ts +0 -53
- package/dist/types/tracker.d.ts +0 -43
- package/dist/types/typed.d.ts +0 -15
- package/dist/types/types.d.ts +0 -26
- package/dist/types/udf.d.ts +0 -75
- package/dist/types/window.d.ts +0 -52
package/src/jobs.js
CHANGED
|
@@ -32,7 +32,8 @@
|
|
|
32
32
|
* timer, and its abandoned job recovers by lease expiry (§5).
|
|
33
33
|
*/
|
|
34
34
|
|
|
35
|
-
import { chain } from './driver.js';
|
|
35
|
+
import { chain, attempt } from './driver.js';
|
|
36
|
+
import { DbCompileError, DbRuntimeError } from './errors.js';
|
|
36
37
|
|
|
37
38
|
export const JOBS_TABLE = '_jaren_jobs';
|
|
38
39
|
export const JOB_CHECKPOINTS_TABLE = '_jaren_job_checkpoints';
|
|
@@ -147,12 +148,25 @@ export function createJobEngine(options) {
|
|
|
147
148
|
const random = options.random ?? Math.random;
|
|
148
149
|
const defaults = { ...JOB_DEFAULTS, ...options.defaults };
|
|
149
150
|
|
|
151
|
+
/** Every queue statement failure rides the store's own wrap (§9):
|
|
152
|
+
* a read-only file, a locked database, a constraint — never the
|
|
153
|
+
* driver's raw error. */
|
|
154
|
+
const wrapJobs = (error) => (typeof error?.code === 'string' && error.code.startsWith('JD')
|
|
155
|
+
? error
|
|
156
|
+
: new DbRuntimeError('JD2005',
|
|
157
|
+
`the database rejected the operation: ${error?.message ?? String(error)}`,
|
|
158
|
+
{ docPath: '/jobs', collection: JOBS_TABLE, cause: error }));
|
|
150
159
|
/** @type {Map<string, any>} */
|
|
151
160
|
const statements = new Map();
|
|
152
161
|
const prepared = (key, sql) => {
|
|
153
162
|
let statement = statements.get(key);
|
|
154
163
|
if (statement === undefined) {
|
|
155
|
-
|
|
164
|
+
const raw = connection.prepare(sql);
|
|
165
|
+
statement = {
|
|
166
|
+
run: (params) => attempt(() => raw.run(params), wrapJobs),
|
|
167
|
+
get: (params) => attempt(() => raw.get(params), wrapJobs),
|
|
168
|
+
all: (params) => attempt(() => raw.all(params), wrapJobs),
|
|
169
|
+
};
|
|
156
170
|
statements.set(key, statement);
|
|
157
171
|
}
|
|
158
172
|
return statement;
|
|
@@ -164,13 +178,27 @@ export function createJobEngine(options) {
|
|
|
164
178
|
for (const wake of [...wakers]) wake();
|
|
165
179
|
};
|
|
166
180
|
|
|
167
|
-
|
|
181
|
+
// the tables are created here, or refused here: a read-only store
|
|
182
|
+
// leaked the driver's "attempt to write a readonly database"
|
|
183
|
+
const ready = attempt(() => connection.exec(CREATE_JOBS), (error) => new DbCompileError('JD0002',
|
|
184
|
+
`the job tables could not be created (${error?.message ?? String(error)}) — `
|
|
185
|
+
+ 'a read-only store creates nothing; open it read-write once, or without jobs',
|
|
186
|
+
'/jobs', error));
|
|
168
187
|
|
|
169
188
|
const enqueue = (kind, payload, enqueueOptions) => {
|
|
170
189
|
if (typeof kind !== 'string' || kind === '') {
|
|
171
190
|
throw new TypeError('enqueue: "kind" must be a non-empty string');
|
|
172
191
|
}
|
|
173
192
|
const id = enqueueOptions?.id ?? crypto.randomUUID();
|
|
193
|
+
// a `runAt` that is not a number stored as NaN and left the job
|
|
194
|
+
// pending forever; a Date could not even be bound
|
|
195
|
+
if (enqueueOptions?.runAt !== undefined && !Number.isFinite(enqueueOptions.runAt)) {
|
|
196
|
+
throw new TypeError('enqueue: "runAt" is an epoch in milliseconds (a finite number)');
|
|
197
|
+
}
|
|
198
|
+
if (enqueueOptions?.maxAttempts !== undefined
|
|
199
|
+
&& !(Number.isInteger(enqueueOptions.maxAttempts) && enqueueOptions.maxAttempts >= 1)) {
|
|
200
|
+
throw new TypeError('enqueue: "maxAttempts" is a positive integer');
|
|
201
|
+
}
|
|
174
202
|
const at = now();
|
|
175
203
|
return chain(prepared('enqueue', `INSERT INTO "${JOBS_TABLE}"
|
|
176
204
|
(id, kind, payload, state, run_at, max_attempts, created_at, updated_at)
|
|
@@ -341,8 +369,12 @@ export function createJobEngine(options) {
|
|
|
341
369
|
const kinds = Object.keys(handlers);
|
|
342
370
|
const owner = workerOptions.owner ?? crypto.randomUUID();
|
|
343
371
|
const concurrency = workerOptions.concurrency ?? 1;
|
|
372
|
+
if (!(Number.isInteger(concurrency) && concurrency >= 1)) {
|
|
373
|
+
// `Array.from({ length: 0 })` started a worker that never claimed
|
|
374
|
+
throw new TypeError('createWorker: "concurrency" is a positive integer');
|
|
375
|
+
}
|
|
344
376
|
const pollInterval = workerOptions.pollInterval ?? defaults.pollInterval;
|
|
345
|
-
const stats = { claims: 0, completions: 0, failures: 0, polls: 0, wakes: 0 };
|
|
377
|
+
const stats = { claims: 0, completions: 0, failures: 0, polls: 0, wakes: 0, claimErrors: 0 };
|
|
346
378
|
|
|
347
379
|
let running = false;
|
|
348
380
|
/** @type {Promise<void>[]} */
|
|
@@ -448,7 +480,10 @@ export function createJobEngine(options) {
|
|
|
448
480
|
kinds, owner, leaseMs: workerOptions.leaseMs }));
|
|
449
481
|
}
|
|
450
482
|
catch {
|
|
451
|
-
|
|
483
|
+
// a storage failure backs off to the poll — COUNTED, so a
|
|
484
|
+
// worker on a read-only store is not silently idle forever
|
|
485
|
+
stats.claimErrors += 1;
|
|
486
|
+
job = undefined;
|
|
452
487
|
}
|
|
453
488
|
if (!running || loopSession.cancelled) return;
|
|
454
489
|
if (job === undefined) {
|
package/src/live-time.js
ADDED
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/**
|
|
3
|
+
* @file Event-time live views (LIVE-FORMAT §13): a `$resample` or
|
|
4
|
+
* `$rolling` document over a collection, maintained against an
|
|
5
|
+
* explicit watermark.
|
|
6
|
+
*
|
|
7
|
+
* There is no clock in this file, and there is none anywhere under it.
|
|
8
|
+
* A live view over time needs to know what "now" is — which bucket is
|
|
9
|
+
* still open, which reading counts as late — and the only honest
|
|
10
|
+
* source of that is the host, because the machine's clock is a
|
|
11
|
+
* different quantity from the instant a reading carries. So the
|
|
12
|
+
* watermark ARRIVES: it is a finite epoch supplied at registration and
|
|
13
|
+
* moved forward by `advance()`, it never goes backwards, and a test can
|
|
14
|
+
* put it wherever the story needs it without waiting for a timer.
|
|
15
|
+
*
|
|
16
|
+
* What the maintenance actually does:
|
|
17
|
+
*
|
|
18
|
+
* - **A bucket view keeps its rows by bucket.** A write touches one
|
|
19
|
+
* bucket (two, when it moves a reading across a boundary), and only
|
|
20
|
+
* those are folded again — through `resampleSeries` itself, over that
|
|
21
|
+
* bucket's own rows, so the aggregate is the kernel's and cannot
|
|
22
|
+
* drift from what a fresh query would answer.
|
|
23
|
+
* - **A rolling view keeps its rows in instant order.** A write at `t`
|
|
24
|
+
* can only change the windows ending in `[t, t + width)`, so exactly
|
|
25
|
+
* that stretch is recomputed — again by the kernel, over the slice
|
|
26
|
+
* that stretch can see.
|
|
27
|
+
*
|
|
28
|
+
* And what it refuses. A calendar ladder walks a wall clock, a named
|
|
29
|
+
* zone needs host code, `locf`/`linear` couple every bucket to its
|
|
30
|
+
* neighbours, and `first`/`last` name a row by a position a maintained
|
|
31
|
+
* map does not preserve. Each of those re-runs with its own reason
|
|
32
|
+
* rather than being approximated. So does a reading older than the
|
|
33
|
+
* declared lateness: the view re-runs, the emission carries a
|
|
34
|
+
* `lateData` record naming the instant and the boundary, and the row is
|
|
35
|
+
* never quietly folded into a bucket its reader already believed
|
|
36
|
+
* closed.
|
|
37
|
+
*
|
|
38
|
+
* `retention` is the horizon this view claims to work over. It is
|
|
39
|
+
* checked, not assumed: it must cover a whole window plus the lateness
|
|
40
|
+
* the caller allows, which is the span a single repair can read. It is
|
|
41
|
+
* NOT a compaction policy — the maintained state is bounded by
|
|
42
|
+
* `live.maxMaintained` exactly as every other strategy's is, and this
|
|
43
|
+
* file drops nothing that an answer still depends on.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { compileJsonQuery } from '@jarenjs/json/query';
|
|
47
|
+
import { isJsonObject } from '@jarenjs/core/object';
|
|
48
|
+
import { compileBuckets, resampleSeries, rollingSeries, toEpoch } from '@jarenjs/core/series';
|
|
49
|
+
|
|
50
|
+
import { DbCompileError } from './errors.js';
|
|
51
|
+
import { chain } from './driver.js';
|
|
52
|
+
import { singularSelector } from './series.js';
|
|
53
|
+
|
|
54
|
+
/** The closed `eventTime` member set (§13.1). */
|
|
55
|
+
const EVENT_TIME_MEMBERS = Object.freeze(['path', 'watermark', 'allowedLateness', 'retention']);
|
|
56
|
+
|
|
57
|
+
/** The aggregates a maintained state answers exactly. `first`/`last`
|
|
58
|
+
* are absent for the planner's own reason: they name a row by its
|
|
59
|
+
* position in the series, and a per-key map does not keep one. */
|
|
60
|
+
const MAINTAINED_AGGREGATES = Object.freeze(['sum', 'mean', 'min', 'max', 'count']);
|
|
61
|
+
|
|
62
|
+
/** The fill policies an EMPTY bucket can answer on its own. `locf` and
|
|
63
|
+
* `linear` read their neighbours, so one late reading moves buckets it
|
|
64
|
+
* never belonged to — that is a re-run, not a repair. */
|
|
65
|
+
const MAINTAINED_FILLS = Object.freeze(['omit', 'null', 'zero']);
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Validate the `eventTime` option into the record the classifier and
|
|
69
|
+
* the strategies read, or `null` when the caller declared none.
|
|
70
|
+
* @param {any} options - the live options
|
|
71
|
+
* @param {string} collection - for the error's `collection` property
|
|
72
|
+
* @returns {null | { member: string, watermark: number,
|
|
73
|
+
* allowedLateness: number, retention: number }}
|
|
74
|
+
* @throws {DbCompileError} `JD0053` for any member this does not admit
|
|
75
|
+
*/
|
|
76
|
+
export function normalizeEventTime(options, collection) {
|
|
77
|
+
const declared = options?.eventTime;
|
|
78
|
+
if (declared === undefined || declared === null) return null;
|
|
79
|
+
const refuse = (reason) => {
|
|
80
|
+
throw new DbCompileError('JD0053', reason, `/collections/${collection}`);
|
|
81
|
+
};
|
|
82
|
+
if (!isJsonObject(declared))
|
|
83
|
+
refuse('live eventTime is an object with a path, a watermark and a retention');
|
|
84
|
+
for (const name of Object.keys(declared)) {
|
|
85
|
+
if (!EVENT_TIME_MEMBERS.includes(name)) {
|
|
86
|
+
refuse(`live eventTime has no member '${name}' — it admits ${
|
|
87
|
+
EVENT_TIME_MEMBERS.map((m) => `'${m}'`).join(', ')}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const member = singularSelector(declared.path);
|
|
91
|
+
if (member === null)
|
|
92
|
+
refuse("live eventTime.path is a singular row selector naming the instant member, like '$.at'");
|
|
93
|
+
const finite = (value, name) => {
|
|
94
|
+
if (typeof value !== 'number' || !Number.isFinite(value))
|
|
95
|
+
refuse(`live eventTime.${name} is a finite epoch in milliseconds, not ${JSON.stringify(value)}`);
|
|
96
|
+
return value;
|
|
97
|
+
};
|
|
98
|
+
const watermark = finite(declared.watermark, 'watermark');
|
|
99
|
+
const retention = finite(declared.retention ?? NaN, 'retention');
|
|
100
|
+
if (retention <= 0)
|
|
101
|
+
refuse(`live eventTime.retention is a positive span, not ${retention}`);
|
|
102
|
+
const allowedLateness = declared.allowedLateness === undefined
|
|
103
|
+
? 0 : finite(declared.allowedLateness, 'allowedLateness');
|
|
104
|
+
if (allowedLateness < 0)
|
|
105
|
+
refuse(`live eventTime.allowedLateness is not negative, unlike ${allowedLateness}`);
|
|
106
|
+
return { member: /** @type {string} */ (member), watermark, allowedLateness, retention };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The rows behind a series operand, as one document: the bare
|
|
111
|
+
* collection, or the collection under the operand's own `$where`.
|
|
112
|
+
* Anything else — a projection, a second binding, an ordering — is not
|
|
113
|
+
* a shape whose rows a key can be tracked through.
|
|
114
|
+
* @param {any} operand - the operator's first (or right) argument
|
|
115
|
+
* @returns {{ source: any } | null}
|
|
116
|
+
*/
|
|
117
|
+
function operandSource(operand) {
|
|
118
|
+
if (operand === '$[*]') return { source: { $for: { it: '$[*]' }, $return: '$it' } };
|
|
119
|
+
if (!isJsonObject(operand) || !isJsonObject(operand.$for)) return null;
|
|
120
|
+
const names = Object.keys(operand.$for);
|
|
121
|
+
if (names.length !== 1 || operand.$for[names[0]] !== '$[*]') return null;
|
|
122
|
+
const binding = names[0];
|
|
123
|
+
const allowed = new Set(['$for', '$where', '$return']);
|
|
124
|
+
if (!Object.keys(operand).every((key) => allowed.has(key))) return null;
|
|
125
|
+
if (operand.$return !== `$${binding}`) return null;
|
|
126
|
+
return {
|
|
127
|
+
source: {
|
|
128
|
+
$for: { [binding]: '$[*]' },
|
|
129
|
+
...(operand.$where !== undefined ? { $where: operand.$where } : {}),
|
|
130
|
+
$return: `$${binding}`,
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Classify a document as an event-time view, or say why it is not one.
|
|
137
|
+
*
|
|
138
|
+
* Returns `null` when the document does not name `$resample` or
|
|
139
|
+
* `$rolling` over this collection at all — the caller then goes on to
|
|
140
|
+
* §7's ordinary table. Every other outcome is a decision: a maintained
|
|
141
|
+
* description, or `{ strategy: 'rerun', reason }`.
|
|
142
|
+
* @param {any} inner - the unwrapped document
|
|
143
|
+
* @param {boolean} windowed - whether a `$subsequence` wrapped it
|
|
144
|
+
* @param {boolean} keyed
|
|
145
|
+
* @param {null | { member: string, watermark: number,
|
|
146
|
+
* allowedLateness: number, retention: number }} eventTime
|
|
147
|
+
* @returns {any}
|
|
148
|
+
*/
|
|
149
|
+
export function classifyEventTime(inner, windowed, keyed, eventTime) {
|
|
150
|
+
if (!isJsonObject(inner)) return null;
|
|
151
|
+
const keys = Object.keys(inner);
|
|
152
|
+
if (keys.length !== 1) return null;
|
|
153
|
+
const name = keys[0];
|
|
154
|
+
if (name !== '$resample' && name !== '$rolling') return null;
|
|
155
|
+
const args = inner[name];
|
|
156
|
+
if (!Array.isArray(args) || args.length !== 2) return null;
|
|
157
|
+
const operand = operandSource(args[0]);
|
|
158
|
+
if (operand === null) return null;
|
|
159
|
+
|
|
160
|
+
const rerun = (reason) => ({ strategy: 'rerun', reason });
|
|
161
|
+
const spec = args[1];
|
|
162
|
+
if (!isJsonObject(spec)) return null;
|
|
163
|
+
if (eventTime === null) {
|
|
164
|
+
return rerun(`'${name}' — a temporal view maintains event time, and none was declared `
|
|
165
|
+
+ '(live options need an eventTime with a finite watermark)');
|
|
166
|
+
}
|
|
167
|
+
if (windowed) return rerun(`'${name}' — a windowed temporal view re-runs`);
|
|
168
|
+
if (!keyed) return rerun('rows without a document key cannot be tracked');
|
|
169
|
+
|
|
170
|
+
// the instant the state places a row by must be the instant the
|
|
171
|
+
// kernel aggregates it by, or the two would disagree row for row
|
|
172
|
+
const at = spec.at === undefined ? 'at' : singularSelector(spec.at);
|
|
173
|
+
if (at === null || at !== eventTime.member) {
|
|
174
|
+
return rerun(`'${name}' — eventTime.path names '${eventTime.member}' and the spec reads `
|
|
175
|
+
+ `${at === null ? 'a selector this view cannot follow' : `'${at}'`}`);
|
|
176
|
+
}
|
|
177
|
+
if (spec.zone !== undefined && spec.zone !== 'UTC') {
|
|
178
|
+
return rerun(`'${name}' — a named zone resolves through the injected provider, which `
|
|
179
|
+
+ 'maintenance would have to consult per boundary');
|
|
180
|
+
}
|
|
181
|
+
// the kernel reads member NAMES (`at`, `value`); the document spells
|
|
182
|
+
// them as row selectors (`$.at`) — handed the document's spelling, the
|
|
183
|
+
// kernel read `row['$.at']` and every fold died on its first reading
|
|
184
|
+
const valueMember = spec.value === undefined ? null : singularSelector(spec.value);
|
|
185
|
+
if (spec.value !== undefined && valueMember === null) {
|
|
186
|
+
return rerun(`'${name}' — the spec reads a value selector this view cannot follow`);
|
|
187
|
+
}
|
|
188
|
+
const kernelSpec = { ...spec, at: eventTime.member,
|
|
189
|
+
...(valueMember === null ? {} : { value: valueMember }) };
|
|
190
|
+
const aggregate = spec.aggregate ?? 'mean';
|
|
191
|
+
if (!MAINTAINED_AGGREGATES.includes(aggregate)) {
|
|
192
|
+
return rerun(`'${name}' — '${aggregate}' names a row by its position in the series, `
|
|
193
|
+
+ 'which a per-key state does not preserve');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// the ladder (or the window) through the kernel's own compiler, so
|
|
197
|
+
// 'PT1H', 3600000 and 'PT60M' are one width and the default anchor is
|
|
198
|
+
// the kernel's rather than a second guess at it
|
|
199
|
+
const span = (() => {
|
|
200
|
+
try {
|
|
201
|
+
return compileBuckets(name === '$resample' ? spec : spec.width, spec);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return null;
|
|
205
|
+
}
|
|
206
|
+
})();
|
|
207
|
+
if (span === null)
|
|
208
|
+
return rerun(`'${name}' — the temporal kernel refuses this specification`);
|
|
209
|
+
if (span.calendar) {
|
|
210
|
+
return rerun(`'${name}' — a calendar ladder walks a wall clock and a month has no width, `
|
|
211
|
+
+ 'so its boundaries move with the data rather than with arithmetic');
|
|
212
|
+
}
|
|
213
|
+
const covered = span.width + eventTime.allowedLateness;
|
|
214
|
+
if (eventTime.retention < covered) {
|
|
215
|
+
return rerun(`'${name}' — a retention of ${eventTime.retention} ms does not cover `
|
|
216
|
+
+ `${covered} ms of window plus allowed lateness, so a repair could read outside `
|
|
217
|
+
+ 'the horizon this view claims');
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (name === '$rolling') {
|
|
221
|
+
const minPeriods = spec.minPeriods ?? 1;
|
|
222
|
+
if (!Number.isInteger(minPeriods) || minPeriods < 1)
|
|
223
|
+
return rerun("'$rolling' — the temporal kernel refuses this specification");
|
|
224
|
+
return {
|
|
225
|
+
strategy: 'rolling',
|
|
226
|
+
source: operand.source,
|
|
227
|
+
spec: kernelSpec,
|
|
228
|
+
member: eventTime.member,
|
|
229
|
+
width: span.width,
|
|
230
|
+
eventTime,
|
|
231
|
+
deps: { whole: true, members: new Set() },
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const fill = spec.fill ?? 'omit';
|
|
236
|
+
if (!MAINTAINED_FILLS.includes(fill)) {
|
|
237
|
+
return rerun(`'$resample' — '${fill}' fills an empty bucket from its neighbours, so one `
|
|
238
|
+
+ 'late reading moves buckets it never belonged to');
|
|
239
|
+
}
|
|
240
|
+
const ladder = span;
|
|
241
|
+
const start = boundOf(spec.start);
|
|
242
|
+
const end = boundOf(spec.end);
|
|
243
|
+
if (start === false || end === false || (start !== null && end !== null && !(start < end)))
|
|
244
|
+
return rerun("'$resample' — the temporal kernel refuses this specification");
|
|
245
|
+
return {
|
|
246
|
+
strategy: 'bucket',
|
|
247
|
+
source: operand.source,
|
|
248
|
+
spec: kernelSpec,
|
|
249
|
+
member: eventTime.member,
|
|
250
|
+
ladder,
|
|
251
|
+
fill,
|
|
252
|
+
aggregate,
|
|
253
|
+
start,
|
|
254
|
+
end,
|
|
255
|
+
eventTime,
|
|
256
|
+
deps: { whole: true, members: new Set() },
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* A window bound as an epoch, `null` when absent, `false` when it names
|
|
262
|
+
* no instant (the kernel's own refusal, asked before a plan exists).
|
|
263
|
+
* @param {any} value
|
|
264
|
+
* @returns {number | null | false}
|
|
265
|
+
*/
|
|
266
|
+
function boundOf(value) {
|
|
267
|
+
if (value === undefined) return null;
|
|
268
|
+
try {
|
|
269
|
+
return toEpoch(value);
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
return false;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* The shared half of both event-time strategies: the watermark, the
|
|
278
|
+
* lateness boundary, per-key row bookkeeping and the re-run a too-late
|
|
279
|
+
* reading forces.
|
|
280
|
+
* @param {any} description
|
|
281
|
+
* @param {any} context
|
|
282
|
+
*/
|
|
283
|
+
function eventTimeBase(description, context) {
|
|
284
|
+
const { member } = description;
|
|
285
|
+
const evaluate = compileJsonQuery([description.source]);
|
|
286
|
+
const stats = { lateData: 0, reruns: 0, recomputes: 0 };
|
|
287
|
+
let watermark = description.eventTime.watermark;
|
|
288
|
+
|
|
289
|
+
/** The instant a row is late BEFORE. */
|
|
290
|
+
const boundary = () => watermark - description.eventTime.allowedLateness;
|
|
291
|
+
|
|
292
|
+
/** The row this document contributes, or `undefined` for none. */
|
|
293
|
+
const rowOf = (doc) => {
|
|
294
|
+
if (doc === undefined) return undefined;
|
|
295
|
+
const items = /** @type {any[]} */ (evaluate([doc], context.externals));
|
|
296
|
+
return items.length === 0 ? undefined : items[0];
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
/** The instant a contributed row carries, through the kernel's own
|
|
300
|
+
* reader so a live view refuses exactly where a query would. */
|
|
301
|
+
const instantOf = (row) => toEpoch(row[member]);
|
|
302
|
+
|
|
303
|
+
const advance = (next) => {
|
|
304
|
+
if (typeof next !== 'number' || !Number.isFinite(next))
|
|
305
|
+
throw new TypeError(`a watermark is a finite epoch in milliseconds, not ${next}`);
|
|
306
|
+
if (next < watermark) {
|
|
307
|
+
throw new TypeError(
|
|
308
|
+
`a watermark only advances: ${next} is behind the current ${watermark}`);
|
|
309
|
+
}
|
|
310
|
+
watermark = next;
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
return {
|
|
314
|
+
stats,
|
|
315
|
+
rowOf,
|
|
316
|
+
instantOf,
|
|
317
|
+
advance,
|
|
318
|
+
boundary,
|
|
319
|
+
watermarkOf: () => watermark,
|
|
320
|
+
/** The `lateData` record an emission carries when a reading landed
|
|
321
|
+
* behind the boundary. */
|
|
322
|
+
late: (at, key) => ({
|
|
323
|
+
reason: 'late-data',
|
|
324
|
+
at,
|
|
325
|
+
key,
|
|
326
|
+
watermark,
|
|
327
|
+
allowedLateness: description.eventTime.allowedLateness,
|
|
328
|
+
boundary: boundary(),
|
|
329
|
+
}),
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* `$resample` over a fixed ladder: one maintained fold per bucket.
|
|
335
|
+
* @param {any} description
|
|
336
|
+
* @param {any} context
|
|
337
|
+
*/
|
|
338
|
+
export function bucketStrategy(description, context) {
|
|
339
|
+
const { ladder, fill, aggregate, start, end, spec } = description;
|
|
340
|
+
const base = eventTimeBase(description, context);
|
|
341
|
+
|
|
342
|
+
/** @type {Map<string, { at: number, bucket: number, row: any }>} */
|
|
343
|
+
const placed = new Map();
|
|
344
|
+
/** @type {Map<number, Map<string, any>>} bucket start → its rows */
|
|
345
|
+
const rows = new Map();
|
|
346
|
+
/** @type {Map<number, { at: number, value: number|null, count: number }>} */
|
|
347
|
+
const folded = new Map();
|
|
348
|
+
|
|
349
|
+
/** Is this instant inside the view's own half-open window? */
|
|
350
|
+
const inWindow = (at) => (start === null || at >= start) && (end === null || at < end);
|
|
351
|
+
|
|
352
|
+
/** Re-fold one bucket through the kernel — the same call, over the
|
|
353
|
+
* same rows, that a fresh query would make over this stretch. */
|
|
354
|
+
const refold = (bucket) => {
|
|
355
|
+
base.stats.recomputes += 1;
|
|
356
|
+
const held = rows.get(bucket);
|
|
357
|
+
if (held === undefined || held.size === 0) {
|
|
358
|
+
rows.delete(bucket);
|
|
359
|
+
folded.delete(bucket);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const out = resampleSeries([...held.values()],
|
|
363
|
+
{ ...spec, start: bucket, end: bucket + ladder.width, fill: 'null' });
|
|
364
|
+
folded.set(bucket, out[0]);
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
/** Put a document's row where it belongs, reporting the buckets that
|
|
368
|
+
* moved. `null` rows out of the window and rows with no contribution. */
|
|
369
|
+
const place = (key, row, touched) => {
|
|
370
|
+
const previous = placed.get(key);
|
|
371
|
+
if (previous !== undefined) {
|
|
372
|
+
rows.get(previous.bucket)?.delete(key);
|
|
373
|
+
touched.add(previous.bucket);
|
|
374
|
+
placed.delete(key);
|
|
375
|
+
}
|
|
376
|
+
if (row === undefined) return;
|
|
377
|
+
const at = base.instantOf(row);
|
|
378
|
+
if (!inWindow(at)) return;
|
|
379
|
+
const bucket = ladder.startOf(ladder.indexOf(at));
|
|
380
|
+
let held = rows.get(bucket);
|
|
381
|
+
if (held === undefined) {
|
|
382
|
+
held = new Map();
|
|
383
|
+
rows.set(bucket, held);
|
|
384
|
+
}
|
|
385
|
+
held.set(key, row);
|
|
386
|
+
placed.set(key, { at, bucket, row });
|
|
387
|
+
touched.add(bucket);
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
/** The result rows: the maintained folds, and — under a fill policy —
|
|
391
|
+
* the empty positions of the ladder between them. */
|
|
392
|
+
const emit = () => {
|
|
393
|
+
const starts = [...folded.keys()].sort((a, b) => a - b);
|
|
394
|
+
if (fill === 'omit') return starts.map((at) => folded.get(at));
|
|
395
|
+
if (starts.length === 0 && (start === null || end === null)) return [];
|
|
396
|
+
const empty = aggregate === 'count' ? 0 : (fill === 'zero' ? 0 : null);
|
|
397
|
+
const from = start === null ? starts[0] : start;
|
|
398
|
+
const last = end === null ? starts[starts.length - 1] : null;
|
|
399
|
+
const out = [];
|
|
400
|
+
let index = ladder.indexOf(from);
|
|
401
|
+
let at = ladder.startOf(index);
|
|
402
|
+
while (end === null ? at <= /** @type {number} */ (last) : at < end) {
|
|
403
|
+
out.push(folded.get(at) ?? { at, value: empty, count: 0 });
|
|
404
|
+
index += 1;
|
|
405
|
+
at = ladder.startOf(index);
|
|
406
|
+
}
|
|
407
|
+
return out;
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
return {
|
|
411
|
+
advance: base.advance,
|
|
412
|
+
stats: () => ({ ...base.stats, watermark: base.watermarkOf() }),
|
|
413
|
+
entries: () => placed.size + folded.size,
|
|
414
|
+
init: () => chain(context.execute([description.source], { externals: context.externals }),
|
|
415
|
+
(docs) => {
|
|
416
|
+
const touched = new Set();
|
|
417
|
+
for (const doc of /** @type {any[]} */ (docs))
|
|
418
|
+
place(context.keyOf(doc), base.rowOf(doc), touched);
|
|
419
|
+
for (const bucket of touched) refold(bucket);
|
|
420
|
+
return emit();
|
|
421
|
+
}),
|
|
422
|
+
apply(record) {
|
|
423
|
+
const touched = context.touchedKeys(record, description.deps);
|
|
424
|
+
if (touched === null) return null;
|
|
425
|
+
/** @type {any} */
|
|
426
|
+
let late = null;
|
|
427
|
+
const moved = new Set();
|
|
428
|
+
for (const [key, change] of touched) {
|
|
429
|
+
const doc = change.kind === 'delete' ? undefined
|
|
430
|
+
: change.kind === 'insert' ? change.doc : context.readRow(key);
|
|
431
|
+
const row = base.rowOf(doc);
|
|
432
|
+
const before = placed.get(key);
|
|
433
|
+
const after = row === undefined ? null : base.instantOf(row);
|
|
434
|
+
const boundary = base.boundary();
|
|
435
|
+
const behind = [before?.at, after].find(
|
|
436
|
+
(at) => typeof at === 'number' && inWindow(at) && at < boundary);
|
|
437
|
+
if (behind !== undefined) {
|
|
438
|
+
late = base.late(behind, key);
|
|
439
|
+
break;
|
|
440
|
+
}
|
|
441
|
+
place(key, row, moved);
|
|
442
|
+
}
|
|
443
|
+
if (late !== null) return { rebuild: true, late };
|
|
444
|
+
if (moved.size === 0) return null;
|
|
445
|
+
for (const bucket of moved) refold(bucket);
|
|
446
|
+
return { rows: emit() };
|
|
447
|
+
},
|
|
448
|
+
/** A re-run rebuilds the whole state from the store — the only
|
|
449
|
+
* answer to a reading the maintained state cannot place. */
|
|
450
|
+
rebuild() {
|
|
451
|
+
base.stats.lateData += 1;
|
|
452
|
+
base.stats.reruns += 1;
|
|
453
|
+
placed.clear();
|
|
454
|
+
rows.clear();
|
|
455
|
+
folded.clear();
|
|
456
|
+
return this.init();
|
|
457
|
+
},
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* `$rolling` over a fixed width: one output per input instant, with
|
|
463
|
+
* only the stretch a write can reach recomputed.
|
|
464
|
+
* @param {any} description
|
|
465
|
+
* @param {any} context
|
|
466
|
+
*/
|
|
467
|
+
export function rollingStrategy(description, context) {
|
|
468
|
+
const { spec, width } = description;
|
|
469
|
+
const base = eventTimeBase(description, context);
|
|
470
|
+
|
|
471
|
+
/** @type {Map<string, { at: number, row: any }>} */
|
|
472
|
+
const placed = new Map();
|
|
473
|
+
/** Rows in instant order; ties keep insertion order, which the seven
|
|
474
|
+
* maintained aggregates cannot tell apart. @type {any[]} */
|
|
475
|
+
let ordered = [];
|
|
476
|
+
/** One output per row of `ordered`. @type {any[]} */
|
|
477
|
+
let out = [];
|
|
478
|
+
|
|
479
|
+
const compareAt = (a, b) => base.instantOf(a) - base.instantOf(b);
|
|
480
|
+
|
|
481
|
+
/** The first position whose instant is at or after `at`. */
|
|
482
|
+
const lowerBound = (at) => {
|
|
483
|
+
let lo = 0;
|
|
484
|
+
let hi = ordered.length;
|
|
485
|
+
while (lo < hi) {
|
|
486
|
+
const mid = (lo + hi) >> 1;
|
|
487
|
+
if (base.instantOf(ordered[mid]) < at) lo = mid + 1;
|
|
488
|
+
else hi = mid;
|
|
489
|
+
}
|
|
490
|
+
return lo;
|
|
491
|
+
};
|
|
492
|
+
/** The first position whose instant is after `at`. */
|
|
493
|
+
const upperBound = (at) => {
|
|
494
|
+
let lo = 0;
|
|
495
|
+
let hi = ordered.length;
|
|
496
|
+
while (lo < hi) {
|
|
497
|
+
const mid = (lo + hi) >> 1;
|
|
498
|
+
if (base.instantOf(ordered[mid]) <= at) lo = mid + 1;
|
|
499
|
+
else hi = mid;
|
|
500
|
+
}
|
|
501
|
+
return lo;
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* Recompute every window ending in `[from, to + width)` — the whole
|
|
506
|
+
* reach of a write at any instant in `[from, to]` — over the slice
|
|
507
|
+
* those windows can see. A window opens EXCLUSIVELY at `at - width`,
|
|
508
|
+
* so the slice starts one position past `from - width` and the
|
|
509
|
+
* kernel's answer for a row inside it is the answer it would give
|
|
510
|
+
* over the entire series.
|
|
511
|
+
*/
|
|
512
|
+
const repair = (from, to) => {
|
|
513
|
+
base.stats.recomputes += 1;
|
|
514
|
+
const lo = upperBound(from - width);
|
|
515
|
+
const hi = lowerBound(to + width);
|
|
516
|
+
const answers = rollingSeries(ordered.slice(lo, hi), spec);
|
|
517
|
+
for (let i = lowerBound(from); i < hi; i++) out[i] = answers[i - lo];
|
|
518
|
+
};
|
|
519
|
+
|
|
520
|
+
const removeRow = (key) => {
|
|
521
|
+
const previous = placed.get(key);
|
|
522
|
+
if (previous === undefined) return null;
|
|
523
|
+
const at = lowerBound(previous.at);
|
|
524
|
+
for (let i = at; i < ordered.length; i++) {
|
|
525
|
+
if (ordered[i] === previous.row) {
|
|
526
|
+
ordered.splice(i, 1);
|
|
527
|
+
out.splice(i, 1);
|
|
528
|
+
break;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
placed.delete(key);
|
|
532
|
+
return previous.at;
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
const insertRow = (key, row) => {
|
|
536
|
+
const at = base.instantOf(row);
|
|
537
|
+
const position = upperBound(at);
|
|
538
|
+
ordered.splice(position, 0, row);
|
|
539
|
+
out.splice(position, 0, null);
|
|
540
|
+
placed.set(key, { at, row });
|
|
541
|
+
return at;
|
|
542
|
+
};
|
|
543
|
+
|
|
544
|
+
return {
|
|
545
|
+
advance: base.advance,
|
|
546
|
+
stats: () => ({ ...base.stats, watermark: base.watermarkOf() }),
|
|
547
|
+
entries: () => placed.size,
|
|
548
|
+
init: () => chain(context.execute([description.source], { externals: context.externals }),
|
|
549
|
+
(docs) => {
|
|
550
|
+
ordered = [];
|
|
551
|
+
for (const doc of /** @type {any[]} */ (docs)) {
|
|
552
|
+
const row = base.rowOf(doc);
|
|
553
|
+
if (row === undefined) continue;
|
|
554
|
+
placed.set(context.keyOf(doc), { at: base.instantOf(row), row });
|
|
555
|
+
ordered.push(row);
|
|
556
|
+
}
|
|
557
|
+
ordered.sort(compareAt);
|
|
558
|
+
out = rollingSeries(ordered, spec);
|
|
559
|
+
return out.slice();
|
|
560
|
+
}),
|
|
561
|
+
apply(record) {
|
|
562
|
+
const touched = context.touchedKeys(record, description.deps);
|
|
563
|
+
if (touched === null) return null;
|
|
564
|
+
/** @type {any} */
|
|
565
|
+
let late = null;
|
|
566
|
+
let from = Infinity;
|
|
567
|
+
let to = -Infinity;
|
|
568
|
+
const moves = [];
|
|
569
|
+
for (const [key, change] of touched) {
|
|
570
|
+
const doc = change.kind === 'delete' ? undefined
|
|
571
|
+
: change.kind === 'insert' ? change.doc : context.readRow(key);
|
|
572
|
+
const row = base.rowOf(doc);
|
|
573
|
+
const before = placed.get(key)?.at;
|
|
574
|
+
const after = row === undefined ? undefined : base.instantOf(row);
|
|
575
|
+
const boundary = base.boundary();
|
|
576
|
+
const behind = [before, after].find(
|
|
577
|
+
(at) => typeof at === 'number' && at < boundary);
|
|
578
|
+
if (behind !== undefined) {
|
|
579
|
+
late = base.late(behind, key);
|
|
580
|
+
break;
|
|
581
|
+
}
|
|
582
|
+
moves.push({ key, row, before, after });
|
|
583
|
+
}
|
|
584
|
+
if (late !== null) return { rebuild: true, late };
|
|
585
|
+
for (const move of moves) {
|
|
586
|
+
for (const at of [move.before, move.after]) {
|
|
587
|
+
if (typeof at !== 'number') continue;
|
|
588
|
+
if (at < from) from = at;
|
|
589
|
+
if (at > to) to = at;
|
|
590
|
+
}
|
|
591
|
+
removeRow(move.key);
|
|
592
|
+
if (move.row !== undefined) insertRow(move.key, move.row);
|
|
593
|
+
}
|
|
594
|
+
if (from === Infinity) return null;
|
|
595
|
+
repair(from, to);
|
|
596
|
+
return { rows: out.slice() };
|
|
597
|
+
},
|
|
598
|
+
rebuild() {
|
|
599
|
+
base.stats.lateData += 1;
|
|
600
|
+
base.stats.reruns += 1;
|
|
601
|
+
placed.clear();
|
|
602
|
+
return this.init();
|
|
603
|
+
},
|
|
604
|
+
};
|
|
605
|
+
}
|