@jarenjs/db 0.56.0 → 0.67.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/ARCHITECTURE.md +412 -56
  2. package/README.md +600 -57
  3. package/docs/HOSTS.md +269 -0
  4. package/docs/JOBS-FORMAT.md +293 -45
  5. package/docs/LIVE-FORMAT.md +169 -20
  6. package/docs/MIGRATION-FORMAT.md +142 -17
  7. package/docs/MODEL-FORMAT.md +752 -64
  8. package/docs/REPLICATION-FORMAT.md +208 -0
  9. package/package.json +21 -7
  10. package/schemas/jaren-model.draft-07.schema.json +224 -162
  11. package/schemas/jaren-model.schema.json +224 -162
  12. package/schemas/jaren-replication-snapshot.draft-07.schema.json +83 -0
  13. package/schemas/jaren-replication-snapshot.schema.json +83 -0
  14. package/schemas/jaren-replication.draft-07.schema.json +82 -0
  15. package/schemas/jaren-replication.schema.json +82 -0
  16. package/src/algebra.js +227 -9
  17. package/src/backup.js +161 -0
  18. package/src/cancellation.js +48 -0
  19. package/src/capture.js +230 -47
  20. package/src/cli.js +165 -59
  21. package/src/cursor.js +417 -0
  22. package/src/dag-job.js +154 -21
  23. package/src/ddl.js +102 -8
  24. package/src/dialect.js +268 -113
  25. package/src/dialects/expression-read.js +158 -0
  26. package/src/dialects/postgres.js +618 -0
  27. package/src/dialects/rtree-ddl.js +129 -0
  28. package/src/dialects/sqlite.js +244 -11
  29. package/src/document-files.js +311 -0
  30. package/src/document-steps.js +422 -0
  31. package/src/documents.js +335 -0
  32. package/src/driver.js +448 -61
  33. package/src/drivers/bun.js +37 -1
  34. package/src/drivers/indexeddb-snapshot.js +149 -0
  35. package/src/drivers/node-pool.js +11 -0
  36. package/src/drivers/node-worker-endpoint.js +105 -0
  37. package/src/drivers/node-worker.js +204 -0
  38. package/src/drivers/node.js +41 -7
  39. package/src/drivers/postgres.js +331 -0
  40. package/src/drivers/wasm-oo1.js +97 -0
  41. package/src/drivers/wasm-session.js +67 -0
  42. package/src/drivers/wasm.js +17 -83
  43. package/src/drivers/worker-pool.js +183 -0
  44. package/src/drivers/worker-protocol.js +79 -0
  45. package/src/drivers/worker-queue.js +60 -0
  46. package/src/emit.js +339 -48
  47. package/src/entity.js +20 -22
  48. package/src/errors.js +430 -19
  49. package/src/expression.js +284 -0
  50. package/src/graph.js +64 -8
  51. package/src/index.js +48 -17
  52. package/src/introspect.js +583 -0
  53. package/src/jobs.js +843 -107
  54. package/src/json-bytes.js +58 -0
  55. package/src/live-join.js +250 -0
  56. package/src/live-nested.js +120 -0
  57. package/src/live.js +18 -4
  58. package/src/logical-rows.js +90 -0
  59. package/src/maintenance.js +175 -0
  60. package/src/migrate.js +248 -181
  61. package/src/model.js +68 -0
  62. package/src/plan.js +1119 -138
  63. package/src/pragmas.js +314 -0
  64. package/src/profile.js +151 -3
  65. package/src/query.js +1634 -323
  66. package/src/replication-format.js +115 -0
  67. package/src/replication.js +332 -0
  68. package/src/residual.js +17 -0
  69. package/src/series.js +12 -4
  70. package/src/store.js +1567 -273
  71. package/src/tracker.js +203 -29
  72. package/src/udf.js +88 -7
  73. package/types/index.d.ts +1158 -27
  74. package/types/node-pool.d.ts +28 -0
  75. package/types/node-worker.d.ts +54 -0
  76. package/types/node.d.ts +69 -2
  77. package/types/postgres.d.ts +46 -0
  78. package/types/typed.d.ts +27 -4
  79. package/types/wasm.d.ts +14 -0
package/src/query.js CHANGED
@@ -32,22 +32,30 @@
32
32
  */
33
33
 
34
34
  import { createSemanticCache } from '@jarenjs/core/cache';
35
- import { compileJsonQuery, analyzeQuery } from '@jarenjs/json/query';
35
+ import { analyzeQuery } from '@jarenjs/json/query';
36
36
 
37
- import { DbCompileError, DbRuntimeError } from './errors.js';
38
- import { chain } from './driver.js';
37
+ import { DbCompileError, DbRuntimeError, wrapDriverError, classifyDriverError } from './errors.js';
38
+ import { chain, attempt, isThenable } from './driver.js';
39
39
  import {
40
40
  planQuery, planEntityQuery, entityShape, planEntityPredicate, entityPathRef,
41
+ isRootScanSource, isEntityRootSource, BIND_REASONS,
41
42
  } from './plan.js';
42
43
  import { emitPlan, emitEntityPlan, createEntityPredicateEmitters, UnrepresentablePath } from './emit.js';
43
- import { selectPlan } from './algebra.js';
44
- import { compileSetResidual, compileRowResidual, sequenceResult } from './residual.js';
44
+ import { selectPlan, conjoin, effectiveOrder, planOrder } from './algebra.js';
45
+ import {
46
+ compileSetResidual, compileRowResidual, compilePackedResidual, sequenceResult,
47
+ } from './residual.js';
48
+ import { createCursor, createSyncCursor, drainPage, utf8Length, PAGE_LIMIT_DEFAULT, rowClassOf } from './cursor.js';
49
+ import { deepFreeze } from '@jarenjs/core/object';
45
50
  import { derivedSlotValue, probeBox, probeVector, columnScore } from './derive.js';
46
51
  import { cutCandidates, identityBatches } from './knn.js';
47
- import { deterministicFragment, registerFragment } from './udf.js';
48
52
  import {
49
- normalizeProfile, translateProfilePredicate,
50
- applyMandatoryPredicate, applyRowBound,
53
+ deterministicFragment, registerFragment, registerAggregateOperator,
54
+ } from './udf.js';
55
+ import { refuseCancelled } from './cancellation.js';
56
+ import {
57
+ normalizeProfile, translateProfilePredicate, assertProfileRoots, memberDenial,
58
+ applyMandatoryPredicate, applyRowBound, SAFE_PROFILE,
51
59
  } from './profile.js';
52
60
 
53
61
  /**
@@ -58,22 +66,31 @@ import {
58
66
  * residual.
59
67
  * @param {number} [bound]
60
68
  * @param {{ functions?: any, extensions?: any } | null} [operators]
61
- * @param {any} [zoneProvider] - D7's injected clock, or absent
69
+ * @param {any} [zoneProvider] - D7's injected zone provider, or absent
70
+ * @param {(() => number) | undefined} [now] - the store's clock, the one a
71
+ * `deadline` is compared against before a call and at every row
72
+ * boundary; the platform's own when the store threads none
62
73
  * @returns {any}
63
74
  */
64
75
  export function createQueryState(bound = undefined, operators = null,
65
- zoneProvider = undefined) {
76
+ zoneProvider = undefined, now = undefined) {
66
77
  return {
67
78
  cache: createSemanticCache(bound ?? 128),
68
79
  counters: { hits: 0, misses: 0, evictions: 0 },
69
80
  /** Fragment identity → the SQL function name registered for it. */
70
81
  registered: new Map(),
82
+ /** Registry operator name → the SQL AGGREGATE registered for it. */
83
+ registeredAggregates: new Map(),
71
84
  operators: operators ?? null,
72
85
  // D7's injected clock: a named zone is host code a database does
73
86
  // not have, so a calendar ladder over one walks in the residual —
74
87
  // and the residual is the caller's OWN document, so the frozen spec
75
88
  // reaches the kernel unchanged rather than being rebuilt in UTC
76
89
  zoneProvider: zoneProvider ?? null,
90
+ // the clock every deadline is read against: the store's runtime
91
+ // record's, so an injected clock and a caller's deadline agree on
92
+ // what time it is — a compiled query never captures the instant
93
+ now: now ?? Date.now,
77
94
  };
78
95
  }
79
96
 
@@ -119,10 +136,28 @@ function bindable(value) {
119
136
  * @param {any} externals
120
137
  * @returns {any}
121
138
  */
122
- function slotValue(slot, externals) {
139
+ function slotValue(slot, externals, anchors = null) {
123
140
  if ('literal' in slot) return slot.literal;
141
+ // a JSON-encoded external: the dialect asked for the value's JSON
142
+ // text because one placeholder has to hold whatever type the member
143
+ // turns out to be, and a statically typed parameter cannot. An
144
+ // ABSENT external stays absent — encoding it as `null` would answer
145
+ // a query the caller never bound instead of raising the engine's own
146
+ // missing-external error
147
+ if ('external' in slot && slot.json === true) {
148
+ const value = externals[slot.external];
149
+ return value === undefined ? undefined : JSON.stringify(value);
150
+ }
124
151
  if ('derived' in slot)
125
152
  return derivedSlotValue(slot.derived, externals[slot.derived.external]);
153
+ if ('typed' in slot) {
154
+ // a typed slot is only ever emitted beside the seek that fills it,
155
+ // so a bind that never resolved the seeks is a defect in the
156
+ // engine, not a value the caller could have got wrong
157
+ if (anchors === null || !(slot.typed.seek in anchors))
158
+ throw new Error(`the seek '${slot.typed.seek}' was not resolved before the bind`);
159
+ return anchors[slot.typed.seek];
160
+ }
126
161
  return externals[slot.external];
127
162
  }
128
163
 
@@ -152,6 +187,150 @@ function externalSlotKinds(slots, rank) {
152
187
  return kinds;
153
188
  }
154
189
 
190
+ /**
191
+ * The per-call preflight every engine runs: a call already aborted
192
+ * issues no statement (`JD2072`); a deadline already passed issues none
193
+ * either (`JD2075`). A deadline is an epoch-millisecond number, checked
194
+ * here and at every row boundary of a cursor — never inside a statement,
195
+ * because the shipped drivers expose no interrupt — against the clock
196
+ * the store was opened with, never the platform's directly.
197
+ * @param {any} options
198
+ * @param {() => number} now - the store's clock
199
+ */
200
+ function requireCallable(options, now) {
201
+ refuseCancelled(options, now, { abortCode: 'JD2072', aborted: 'it ran', passed: 'the call ran' });
202
+ }
203
+
204
+ /**
205
+ * Run a call whose driver failure classes as an int64 overflow through
206
+ * `fallback` instead — the pushed aggregate's coded residual — and let
207
+ * every other failure propagate. Value-or-promise aware.
208
+ * @param {() => any} call
209
+ * @param {() => any} fallback
210
+ * @returns {any}
211
+ */
212
+ function recoverOverflow(call, fallback) {
213
+ let out;
214
+ try {
215
+ out = call();
216
+ }
217
+ catch (error) {
218
+ if (classifyDriverError(error).class === 'overflow') return fallback();
219
+ throw error;
220
+ }
221
+ return isThenable(out)
222
+ ? out.then(undefined, (error) => (classifyDriverError(error).class === 'overflow'
223
+ ? fallback()
224
+ : Promise.reject(error)))
225
+ : out;
226
+ }
227
+
228
+ /** The boundary every engine member answers through: a driver failure
229
+ * arrives classified, anything else as it is.
230
+ * @param {(...args: any[]) => any} fn
231
+ * @param {(error: any) => Error} wrap */
232
+ const bounded = (fn, wrap) => (...args) => attempt(() => fn(...args), wrap);
233
+
234
+ /**
235
+ * The budget provenance `explain()` carries (D7): which profile applied
236
+ * and from where, every bound it imposed — each one the engine COUNTS
237
+ * and enforces — and, by name, the two the driver cannot measure:
238
+ * elapsed statement time and visited rows are empty capability slots on
239
+ * SQLite, so they are reported `unavailable`, never approximated.
240
+ * @param {any} profile - the normalized profile, or null
241
+ * @param {'call' | 'store' | null} source
242
+ * @param {any} capabilities - the connection's capability table
243
+ */
244
+ function budgetOf(profile, source, capabilities) {
245
+ return {
246
+ profile: profile === null ? null : { source, name: profile === SAFE_PROFILE ? 'safe' : 'custom' },
247
+ rows: profile === null ? null : profile.maxRows,
248
+ includedRows: profile === null ? null : profile.maxIncludedRows,
249
+ depth: profile === null ? null : profile.maxDepth,
250
+ bytes: profile === null ? null : profile.maxBytes,
251
+ limits: profile === null ? null : { ...profile.limits },
252
+ scan: profile !== null && profile.refuseFullScan === true ? 'refused-by-shape' : 'unbounded',
253
+ time: capabilities?.statementTimeout === true ? 'enforced' : 'unavailable',
254
+ estimatedRows: capabilities?.rowEstimates === true ? 'available' : 'unavailable',
255
+ };
256
+ }
257
+
258
+ /**
259
+ * The items one projected row answers (a plan whose `project` is a
260
+ * member path): the JSON type decides — an absent member yields no
261
+ * item, a present `null` a null, `true`/`false` the boolean the integer
262
+ * rendering would have lost, anything else the parsed JSON text.
263
+ * @param {any} row
264
+ * @returns {any[]}
265
+ */
266
+ function projectedItems(row) {
267
+ return leafItems(row, '');
268
+ }
269
+
270
+ /**
271
+ * A plan ref's path as the explanation publishes it: member names and
272
+ * array indexes, in order.
273
+ * @param {any} ref
274
+ * @returns {(string | number)[]}
275
+ */
276
+ function segmentsOf(ref) {
277
+ return ref.segments.map((segment) => ('name' in segment ? segment.name : segment.index));
278
+ }
279
+
280
+ /**
281
+ * One projected LEAF of a row, by the suffix its value/type pair was
282
+ * named under: the empty suffix for the single-path plan, `0`, `1`, …
283
+ * for a projection tree's leaves.
284
+ * @param {any} row
285
+ * @param {string} suffix
286
+ * @returns {any[]} the item, or nothing where the member is absent
287
+ */
288
+ function leafItems(row, suffix) {
289
+ const type = row[`t${suffix}`];
290
+ if (type === null || type === undefined) return [];
291
+ const value = row[`v${suffix}`];
292
+ if (type === 'true') return [true];
293
+ if (type === 'false') return [false];
294
+ if (type === 'null') return [null];
295
+ if (type === 'object' || type === 'array') return [JSON.parse(value)];
296
+ if (type === 'text') return [String(value)];
297
+ return [Number(value)];
298
+ }
299
+
300
+ /**
301
+ * The one item a projection TREE answers for one row, rebuilt from the
302
+ * leaves the statement fetched. The engine's own rules decide what an
303
+ * absent leaf does: a member whose value is the empty sequence is
304
+ * OMITTED from its object and SKIPPED in its array — which is why a
305
+ * literal `null` (present in every row) and a path that finds nothing
306
+ * (present in none) cannot share a representation here.
307
+ * @param {any} project - the plan's `{ tree, leaves }`
308
+ * @param {any} row
309
+ * @param {string} [prefix] - what the statement named its value/type
310
+ * pairs: empty on a collection, `'p'` on an entity plan, whose own
311
+ * binding aliases are `t0`, `t1`, … and would collide with a bare one
312
+ * @returns {any[]} exactly one item; a tree always constructs something
313
+ */
314
+ function projectedTreeItems(project, row, prefix = '') {
315
+ const build = (node) => {
316
+ if (node.p === 'lit') return [node.value];
317
+ if (node.p === 'leaf') return leafItems(row, `${prefix}${node.index}`);
318
+ if (node.p === 'object') {
319
+ /** @type {any} */
320
+ const out = {};
321
+ for (const member of node.members) {
322
+ const items = build(member.node);
323
+ if (items.length > 0) out[member.name] = items[0];
324
+ }
325
+ return [out];
326
+ }
327
+ const items = [];
328
+ for (const item of node.items) items.push(...build(item));
329
+ return [items];
330
+ };
331
+ return build(project.tree);
332
+ }
333
+
155
334
  /**
156
335
  * The query engine for one collection.
157
336
  * @param {{ connection: any, state: any, collection: any,
@@ -164,6 +343,10 @@ function externalSlotKinds(slots, rank) {
164
343
  export function createQueryEngine(context) {
165
344
  const { connection, state, collection, physicalPlan } = context;
166
345
  const storeProfile = context.profile ?? null;
346
+ // every root a profile's member allow-list may name: the model's
347
+ // collections and entities, so a typo in the policy is refused rather
348
+ // than applied to nothing
349
+ const roots = context.roots ?? [collection.name];
167
350
  // the store's registered operators (Ring 2): recognised by the planner
168
351
  // as vocabulary, evaluated in the residual, threaded into every
169
352
  // residual compilation here. `null` when the store opened with no
@@ -210,26 +393,57 @@ export function createQueryEngine(context) {
210
393
  * `diverted` counts the calls whose native bucket met a group with no
211
394
  * instant and handed the whole question back to the engine. */
212
395
  const seriesStats = { queries: 0, statements: 0, candidates: 0, results: 0, diverted: 0 };
396
+ /** The PLAIN bind-time diversion counter: calls whose plan was native
397
+ * or row-mode and whose bound external the database could not take (a
398
+ * boolean, a null, a missing name, a region with no box), so the whole
399
+ * collection was read and the engine answered. The k-nearest and the
400
+ * temporal diversions have their own counters; this is the one the
401
+ * ordinary predicate takes, and it is what proves the diversion in
402
+ * production where nobody calls `explain()`. */
403
+ const bindStats = { diverted: 0 };
213
404
  /** The by-identities fetch statements, one per batch size. */
214
405
  const identityFetch = new Map();
215
406
 
216
- /** The `compileJsonQuery` options for an inline residual: the
217
- * profile's engine limits plus the store's registered operators. */
218
- const residualCompileOptions = (limits) => {
219
- const functions = operators?.functions;
220
- const extensions = operators?.extensions;
221
- if (limits === undefined && functions === undefined && extensions === undefined
222
- && zoneProvider === null)
223
- return undefined;
224
- /** @type {any} */
225
- const options = {};
226
- if (limits !== undefined) options.limits = limits;
227
- if (functions !== undefined) options.functions = functions;
228
- if (extensions !== undefined) options.extensions = extensions;
229
- if (zoneProvider !== null) options.zoneProvider = zoneProvider;
230
- return options;
407
+ /** Every driver failure this engine meets, classified under the
408
+ * collection it belongs to. */
409
+ const driverWrap = (/** @type {any} */ error) =>
410
+ wrapDriverError(error, { docPath: collection.docPath, collection: collection.name });
411
+ /**
412
+ * The coded residual for a pushed aggregate that overflowed int64:
413
+ * the engine answers the caller's document over the fetched rows —
414
+ * exactly what an unpushed run always answered — and the entry
415
+ * remembers that it did, for `explain()`. The fetch is a full scan,
416
+ * so a profile that refuses one refuses here too.
417
+ * @param {any} entry @param {any} externals @param {any} document
418
+ */
419
+ const overflowResidual = (entry, externals, document) => {
420
+ if (entry.needsScanCheck) {
421
+ throw profileRefusal(`the profile refuses a full-table scan of '${collection.name}' `
422
+ + '(the pushed aggregate overflowed int64 and the engine would read the whole collection)');
423
+ }
424
+ entry.overflowRuns = (entry.overflowRuns ?? 0) + 1;
425
+ return chain(fullScanOf(entry), (statement) =>
426
+ chain(statement.all(fullScanParams(entry)), (rows) =>
427
+ setResidualOf(entry, document)(rowsToDocs(checkRowBound(entry, rows)), externals)));
231
428
  };
232
429
 
430
+ /**
431
+ * Ring 3 for AGGREGATES: a registry `agg` operator the pack marked
432
+ * `pushable: 'aggregate'` becomes a SQL aggregate over the member's
433
+ * column, folded by the same pure function the residual would call.
434
+ * Two independent gates, never one inferred from the other: the
435
+ * DRIVER must have an aggregate API (bun:sqlite does not), and the
436
+ * pack must have declared the operator poolable that way.
437
+ */
438
+ const aggregateHook = connection.capabilities.aggregateFunctions === true
439
+ && operators !== null && operators.pushableAggregate?.size > 0
440
+ ? (/** @type {string} */ name) => {
441
+ const spec = operators.pushableAggregate.get(name);
442
+ if (spec === undefined) return null;
443
+ return { sql: registerAggregateOperator(connection, state.registeredAggregates, name, spec) };
444
+ }
445
+ : undefined;
446
+
233
447
  const udfHook = connection.capabilities.userFunctions
234
448
  ? (/** @type {any} */ fragment, /** @type {string} */ binding) => {
235
449
  // Ring 3: admit the registry's pushable:'scalar' operators too
@@ -275,23 +489,35 @@ export function createQueryEngine(context) {
275
489
  `the profile does not allow querying collection '${collection.name}'`);
276
490
  }
277
491
 
278
- // no UDF registration under a profile: a foreign document must not
279
- // cause host-side function registration
492
+ // no host-side registration under a profile: a foreign document must
493
+ // not cause one, for a predicate fragment or an aggregate alike
494
+ const registering = profile === null && pushdown;
280
495
  let planned = planQuery(document, shape,
281
- { udf: profile === null && pushdown ? udfHook : undefined });
496
+ { udf: registering ? udfHook : undefined,
497
+ aggregate: registering ? aggregateHook : undefined });
282
498
  if (!pushdown) {
283
499
  planned = {
284
500
  ...planned,
285
501
  plan: null,
286
502
  mode: 'set',
287
- reasons: [{ construct: 'pushdown', reason: 'disabled by the harness switch' }],
503
+ reasons: [{ construct: 'pushdown', reason: BIND_REASONS.pushdown }],
288
504
  rowReturn: null,
289
505
  udfs: [],
290
506
  prefilters: [],
507
+ // the whole collection is fetched and the engine answers: the
508
+ // temporal record says so, whatever index the plan would have used
509
+ series: planned.series === null ? null
510
+ : { ...planned.series, mode: 'engine', index: null, prefix: [] },
291
511
  };
292
512
  }
293
513
 
294
514
  if (profile !== null) {
515
+ // the member allow-list: what the caller may OBTAIN, checked
516
+ // against every member path the document references, before a
517
+ // statement exists
518
+ const denied = memberDenial(profile, collection.name,
519
+ planned.analysis.root, isRootScanSource);
520
+ if (denied !== null) throw profileRefusal(denied);
295
521
  const deps = planned.analysis.dependencies;
296
522
  for (const name of deps.functions) {
297
523
  if (!profile.functions.includes(name))
@@ -353,16 +579,29 @@ export function createQueryEngine(context) {
353
579
  plan,
354
580
  sql: emitted.sql,
355
581
  slots: emitted.slots,
582
+ // the plan's own anchor reads, prepared on first use and kept
583
+ // with it: each one binds a TYPED slot in the statement above
584
+ seeks: (emitted.seeks ?? []).map((seek) => ({ ...seek, statement: null })),
356
585
  externalNames,
357
586
  externalSlotKinds: externalSlotKinds(emitted.slots, plan.rank),
358
587
  // a literal probe is normalized once, here; an external one per
359
588
  // call, from the bound value
360
589
  probe: plan.rank !== null && 'lit' in plan.rank.probe
361
- ? probeVector(plan.rank.probe.lit, plan.rank.dims) : null,
590
+ ? probeVector(plan.rank.probe.lit, plan.rank.alternatives[0].dims) : null,
591
+ // one emitted statement per declared width, prepared on first use
592
+ // and cached with the plan: the bind picks the alternative the
593
+ // probe's own width names, and prepares nothing per call
594
+ rankAlternatives: plan.rank === null ? null
595
+ : plan.rank.alternatives.map((alternative) => {
596
+ const one = emitPlan({ ...plan, rank: { ...plan.rank, alternatives: [alternative] } },
597
+ dialect, physical);
598
+ return { ...alternative, sql: one.sql, slots: one.slots, statement: null };
599
+ }),
362
600
  dependencies: planned.analysis.dependencies,
363
601
  limits: planned.analysis.limits,
364
602
  residualLimits: limits,
365
603
  rowBound: maxRows,
604
+ byteBound: profile === null ? null : profile.maxBytes,
366
605
  needsScanCheck: profile !== null && profile.refuseFullScan === true,
367
606
  scanChecked: false,
368
607
  statement: null,
@@ -385,7 +624,7 @@ export function createQueryEngine(context) {
385
624
  };
386
625
 
387
626
  const statementOf = (entry) => {
388
- if (entry.statement === null) entry.statement = connection.prepare(entry.sql);
627
+ if (entry.statement === null) entry.statement = connection.prepare(entry.sql, { readOnly: true });
389
628
  return entry.statement;
390
629
  };
391
630
  const setResidualOf = (entry, document) => {
@@ -398,8 +637,8 @@ export function createQueryEngine(context) {
398
637
  * whole result sequence into one unambiguous array. */
399
638
  const packedResidualOf = (entry, document) => {
400
639
  if (entry.packedResidual === null) {
401
- const compiled = compileJsonQuery([document], residualCompileOptions(entry.residualLimits));
402
- entry.packedResidual = (candidates, externals) => compiled(candidates, externals);
640
+ entry.packedResidual = compilePackedResidual(document, entry.residualLimits, operators,
641
+ zoneProvider);
403
642
  }
404
643
  return entry.packedResidual;
405
644
  };
@@ -408,27 +647,42 @@ export function createQueryEngine(context) {
408
647
  * profile's mandatory predicate and row bound — a diverted call must
409
648
  * not escape either.
410
649
  */
411
- const fullScanOf = (entry) => {
650
+ const fullScanEmitted = (entry) => {
412
651
  if (entry.fullScanSql === null) {
413
652
  const emitted = emitPlan(entry.fullScanShape(), dialect, physical);
414
653
  entry.fullScanSql = { sql: emitted.sql, slots: emitted.slots, statement: null };
415
654
  }
416
- if (entry.fullScanSql.statement === null)
417
- entry.fullScanSql.statement = connection.prepare(entry.fullScanSql.sql);
418
- return entry.fullScanSql.statement;
655
+ return entry.fullScanSql;
656
+ };
657
+ const fullScanOf = (entry) => {
658
+ const emitted = fullScanEmitted(entry);
659
+ if (emitted.statement === null) emitted.statement = connection.prepare(emitted.sql, { readOnly: true });
660
+ return emitted.statement;
419
661
  };
420
662
  const fullScanParams = (entry) =>
421
- entry.fullScanSql.slots.map((slot) => ('literal' in slot ? slot.literal : null));
663
+ fullScanEmitted(entry).slots.map((slot) => ('literal' in slot ? slot.literal : null));
422
664
 
423
- /** Refuse a fetch that crossed the profile's row bound (JD2007). */
665
+ /** Refuse a fetch that crossed the profile's row bound (JD2007), or
666
+ * a row whose document is larger than its byte bound (JD2076). */
424
667
  const checkRowBound = (entry, rows) => {
425
668
  if (entry.rowBound !== null && rows.length > entry.rowBound) {
426
669
  throw new DbRuntimeError('JD2007',
427
670
  `the fetch crossed the profile's maxRows bound of ${entry.rowBound}`,
428
671
  { docPath: collection.docPath, collection: collection.name });
429
672
  }
673
+ if (entry.byteBound !== null) for (const row of rows) checkByteBound(entry, row);
430
674
  return rows;
431
675
  };
676
+ /** One row's document against the profile's byte bound. */
677
+ const checkByteBound = (entry, row) => {
678
+ if (entry.byteBound === null || typeof row.doc !== 'string') return;
679
+ const bytes = utf8Length(row.doc);
680
+ if (bytes > entry.byteBound) {
681
+ throw new DbRuntimeError('JD2076',
682
+ `an item of ${bytes} serialised bytes exceeds the profile's maxBytes bound of ${entry.byteBound}`,
683
+ { docPath: collection.docPath, collection: collection.name });
684
+ }
685
+ };
432
686
 
433
687
  /** The optional plan-shape refusal: a full-table SCAN of a profiled
434
688
  * collection is refused when the profile says so, verified against
@@ -436,35 +690,116 @@ export function createQueryEngine(context) {
436
690
  const guardScan = (entry) => {
437
691
  if (!entry.needsScanCheck || entry.scanChecked) return null;
438
692
  const eqpParams = entry.slots.map((slot) => ('literal' in slot ? slot.literal : null));
439
- return chain(connection.prepare(dialect.explainQuery(entry.sql)), (statement) =>
693
+ return chain(connection.prepare(dialect.explainQuery(entry.sql), { readOnly: true }), (statement) =>
440
694
  chain(statement.all(eqpParams), (rows) => {
441
- const fullScan = rows.some((row) => {
442
- const detail = String(row.detail);
443
- return detail.startsWith(`SCAN ${physical.table}`)
444
- && !detail.includes('USING INDEX');
445
- });
446
- if (fullScan) {
695
+ const lines = dialect.explainLines(rows);
696
+ if (lines.some((line) => dialect.isFullScan(line, [physical.table]))) {
447
697
  throw profileRefusal(
448
698
  `the profile refuses a full-table scan of '${collection.name}' `
449
- + `(${rows.map((row) => String(row.detail)).join('; ')})`);
699
+ + `(${lines.join('; ')})`);
450
700
  }
451
701
  entry.scanChecked = true;
452
702
  return null;
453
703
  }));
454
704
  };
455
705
 
456
- /** Bind slots against the call's externals. */
457
- const bindParams = (entry, externals) =>
458
- entry.slots.map((slot) => slotValue(slot, externals));
706
+ /**
707
+ * The declared width a probe names, as the emitted alternative that
708
+ * reads it — or `null` when no declared width takes this value, which
709
+ * is the diversion a wrong-width probe has always been. A `null`
710
+ * probe, a probe of the wrong shape and a probe of an undeclared
711
+ * width are one answer here: the database has no column for it.
712
+ * @param {any} entry
713
+ * @param {any} value - the bound probe
714
+ */
715
+ const rankAlternativeFor = (entry, value) =>
716
+ entry.rankAlternatives.find((alternative) =>
717
+ probeVector(value, alternative.dims) !== null) ?? null;
718
+
719
+ /** One alternative's statement, prepared once and kept with the plan. */
720
+ const alternativeStatement = (alternative) => {
721
+ if (alternative.statement === null)
722
+ alternative.statement = connection.prepare(alternative.sql, { readOnly: true });
723
+ return alternative.statement;
724
+ };
459
725
 
460
- /** Must this call divert to the residual? */
461
- const mustDivert = (entry, externals) =>
462
- entry.externalNames.some((name) => {
726
+ /**
727
+ * The anchors this entry's seeks answer, read before the statement
728
+ * that binds them. Each seek is one aggregate read through the same
729
+ * declared index, prepared once and kept with the plan; a seek that
730
+ * finds nothing binds its own probe, which excludes exactly the rows
731
+ * it proved are not there.
732
+ * @param {any} entry
733
+ * @returns {any} value-or-promise of the name → anchor map
734
+ */
735
+ const seekAnchors = (entry) => {
736
+ /** @type {Record<string, any>} */
737
+ const anchors = Object.create(null);
738
+ const next = (i) => {
739
+ if (i >= entry.seeks.length) return anchors;
740
+ const seek = entry.seeks[i];
741
+ if (seek.statement === null) seek.statement = connection.prepare(seek.sql, { readOnly: true });
742
+ return chain(seek.statement, (prepared) =>
743
+ chain(prepared.get(seek.slots.map((slot) => slotValue(slot, {}))), (row) => {
744
+ anchors[seek.name] = anchorValue(seek, row);
745
+ return next(i + 1);
746
+ }));
747
+ };
748
+ return next(0);
749
+ };
750
+
751
+ /**
752
+ * One seek's answer, type-checked before it can reach a statement.
753
+ * The column is of declared type, so the only way a value of another
754
+ * type arrives is a defect below the store (a driver handing back a
755
+ * BigInt, a column written past the declaration) — and a bind that
756
+ * silently compared a number against text would answer WRONG rather
757
+ * than fail, so it is refused here, before the SQL it would bind.
758
+ * @param {any} seek
759
+ * @param {any} row
760
+ */
761
+ const anchorValue = (seek, row) => {
762
+ const value = row === undefined || row === null ? null : row.anchor;
763
+ if (value === null || value === undefined) return seek.fallback;
764
+ const kind = typeof value === 'number' && Number.isFinite(value) ? 'number'
765
+ : typeof value === 'string' ? 'text' : null;
766
+ if (kind !== seek.kind) {
767
+ throw new DbRuntimeError('JD2086',
768
+ `the seek '${seek.name}' declared ${seek.kind} and the database answered `
769
+ + `${typeof value}`, collection.name);
770
+ }
771
+ return value;
772
+ };
773
+
774
+ /** Bind slots against the call's externals, and the seeks' anchors. */
775
+ const bindParams = (entry, externals) =>
776
+ (entry.seeks.length === 0
777
+ ? entry.slots.map((slot) => slotValue(slot, externals))
778
+ : chain(seekAnchors(entry), (anchors) =>
779
+ entry.slots.map((slot) => slotValue(slot, externals, anchors))));
780
+
781
+ /** Run a bound statement: the bind itself may have to READ first. */
782
+ const runAll = (entry, externals, statement) =>
783
+ chain(bindParams(entry, externals), (params) => statement.all(params));
784
+ const runGet = (entry, externals, statement) =>
785
+ chain(bindParams(entry, externals), (params) => statement.get(params));
786
+ /** How many statements one execution of this entry costs. */
787
+ const statementCost = (entry) => 1 + entry.seeks.length;
788
+
789
+ /** The external whose bound value sends this call to the residual —
790
+ * a value the database cannot take, a region with no box, a probe of
791
+ * the wrong width — or `null` when every external binds. */
792
+ const divertingExternal = (entry, externals) =>
793
+ entry.externalNames.find((name) => {
463
794
  const kind = entry.externalSlotKinds.get(name);
464
795
  if (kind === 'derived') return probeBox(externals[name]) === null;
465
- if (kind === 'probe') return probeVector(externals[name], entry.plan.rank.dims) === null;
796
+ // a probe binds when SOME declared width takes it; a width the
797
+ // model does not declare is the diversion it always was
798
+ if (kind === 'probe') return rankAlternativeFor(entry, externals[name]) === null;
466
799
  return !bindable(externals[name]);
467
- });
800
+ }) ?? null;
801
+ /** Must this call divert to the residual? */
802
+ const mustDivert = (entry, externals) => divertingExternal(entry, externals) !== null;
468
803
 
469
804
  const rowsToDocs = (rows) => rows.map((row) => JSON.parse(row.doc));
470
805
 
@@ -499,6 +834,46 @@ export function createQueryEngine(context) {
499
834
  return items;
500
835
  };
501
836
 
837
+ /**
838
+ * The items a GENERAL grouping answers: one per group row, built from
839
+ * the group's keys and aggregates through the plan's own tree. A key
840
+ * comes back with its JSON type beside it, so an absent key leaves
841
+ * its member out exactly as the object constructor does; an aggregate
842
+ * over no values follows the mapping the plan recorded — `0` for a
843
+ * count or a sum, the empty sequence for the other three, which is
844
+ * the ENGINE's answer, not SQL's `NULL`.
845
+ * @param {any} entry
846
+ * @param {any[]} rows
847
+ * @returns {any[]}
848
+ */
849
+ const groupItems = (entry, rows) => {
850
+ const group = entry.plan.group;
851
+ const aggregateItems = (row, index) => {
852
+ const value = row[`a${index}`] ?? null;
853
+ const empty = group.aggregates[index].empty;
854
+ if (value === null) return empty === 'omit' ? [] : [empty === 'zero' ? 0 : null];
855
+ return [value];
856
+ };
857
+ const build = (node, row) => {
858
+ if (node.p === 'lit') return [node.value];
859
+ if (node.p === 'key') return leafItems(row, `k${node.index}`);
860
+ if (node.p === 'agg') return aggregateItems(row, node.index);
861
+ if (node.p === 'object') {
862
+ /** @type {any} */
863
+ const out = {};
864
+ for (const member of node.members) {
865
+ const items = build(member.node, row);
866
+ if (items.length > 0) out[member.name] = items[0];
867
+ }
868
+ return [out];
869
+ }
870
+ const items = [];
871
+ for (const item of node.items) items.push(...build(item, row));
872
+ return [items];
873
+ };
874
+ return rows.flatMap((row) => build(group.tree, row));
875
+ };
876
+
502
877
  /** How many ITEMS an engine result carries (its own shape rule). */
503
878
  const itemCount = (answer) => (answer === undefined ? 0
504
879
  : Array.isArray(answer) ? answer.length : 1);
@@ -520,14 +895,35 @@ export function createQueryEngine(context) {
520
895
  }));
521
896
  };
522
897
 
523
- /** Record one actual execution against the entry and the store. */
898
+ /** Record one actual execution against the entry and the store. A
899
+ * `null` candidate count is a run the database finished on its own —
900
+ * an aggregate — where no row reached the engine and SQLite reports no
901
+ * visited-row count: the slot stays empty rather than estimated. */
524
902
  const countSeries = (entry, statements, candidates, results) => {
525
903
  if (entry.planned.series === null) return;
526
904
  seriesStats.queries++;
527
905
  seriesStats.statements += statements;
528
- seriesStats.candidates += candidates;
906
+ if (candidates !== null) seriesStats.candidates += candidates;
529
907
  seriesStats.results += results;
530
- entry.seriesCounts = { statements, candidates, results };
908
+ entry.seriesCounts = { statements, candidates, results, partial: false };
909
+ };
910
+ /** A cursor's run accounting: counted as it is drained, final when it
911
+ * settles; a mid-iteration `explain()` reads the numbers so far and
912
+ * says so (`partial: true`). */
913
+ const seriesTally = (entry) => {
914
+ if (entry.planned.series === null) return null;
915
+ const live = { statements: statementCost(entry), candidates: 0, results: 0, partial: true };
916
+ entry.seriesCounts = live;
917
+ return {
918
+ row: (items) => {
919
+ live.candidates++;
920
+ live.results += items;
921
+ },
922
+ settle: (opened) => {
923
+ if (entry.seriesCounts === live) entry.seriesCounts = { ...live, partial: false };
924
+ if (opened) countSeries(entry, statementCost(entry), live.candidates, live.results);
925
+ },
926
+ };
531
927
  };
532
928
 
533
929
  /**
@@ -546,7 +942,7 @@ export function createQueryEngine(context) {
546
942
  const batch = batches[i];
547
943
  let statement = identityFetch.get(batch.size);
548
944
  if (statement === undefined) {
549
- statement = connection.prepare(dialect.dml.selectByIdentities(physical, batch.size));
945
+ statement = connection.prepare(dialect.dml.selectByIdentities(physical, batch.size), { readOnly: true });
550
946
  identityFetch.set(batch.size, statement);
551
947
  }
552
948
  return chain(statement, (prepared) => chain(prepared.all(batch.params), (rows) => {
@@ -568,12 +964,15 @@ export function createQueryEngine(context) {
568
964
  */
569
965
  const knnCandidates = (entry, externals) => {
570
966
  const rank = entry.plan.rank;
571
- const probe = entry.probe ?? probeVector(externals[rank.probe.ext], rank.dims);
572
- return chain(statementOf(entry), (statement) =>
573
- chain(statement.all(bindParams(entry, externals)), (rows) => {
967
+ const chosen = 'lit' in rank.probe
968
+ ? entry.rankAlternatives[0]
969
+ : rankAlternativeFor(entry, externals[rank.probe.ext]);
970
+ const probe = entry.probe ?? probeVector(externals[rank.probe.ext], chosen.dims);
971
+ return chain(alternativeStatement(chosen), (statement) =>
972
+ chain(statement.all(chosen.slots.map((slot) => slotValue(slot, externals))), (rows) => {
574
973
  checkRowBound(entry, rows);
575
974
  const scored = rows.map((row) =>
576
- ({ identity: row.rid, score: columnScore(row.vec, rank.dims, probe) }));
975
+ ({ identity: row.rid, score: columnScore(row.vec, chosen.dims, probe) }));
577
976
  const cut = cutCandidates(scored, rank.offset + rank.limit, rank.margin);
578
977
  knnStats.queries++;
579
978
  knnStats.rows += rows.length;
@@ -601,13 +1000,14 @@ export function createQueryEngine(context) {
601
1000
  + '(a bound external the database cannot take diverted the call to the whole collection)');
602
1001
  }
603
1002
  if (entry.planned.mode === 'knn') knnStats.diverted++;
1003
+ else bindStats.diverted++;
604
1004
  return chain(fullScanOf(entry), (statement) =>
605
1005
  chain(statement.all(fullScanParams(entry)), (rows) =>
606
1006
  rowsToDocs(checkRowBound(entry, rows))));
607
1007
  }
608
1008
  if (entry.planned.mode === 'knn') return knnCandidates(entry, externals);
609
1009
  return chain(statementOf(entry), (statement) =>
610
- chain(statement.all(bindParams(entry, externals)), (rows) =>
1010
+ chain(runAll(entry, externals, statement), (rows) =>
611
1011
  rowsToDocs(checkRowBound(entry, rows))));
612
1012
  };
613
1013
 
@@ -619,6 +1019,21 @@ export function createQueryEngine(context) {
619
1019
  return value === null ? undefined : value;
620
1020
  };
621
1021
 
1022
+ /**
1023
+ * The profile one call runs under, with its member allow-list checked
1024
+ * against the model's declared roots — a policy naming a root the
1025
+ * model does not have applies to nothing, which is a policy failing
1026
+ * open, so it is refused here rather than at the first query that
1027
+ * happens to name that root.
1028
+ * @param {any} options
1029
+ */
1030
+ const resolveProfile = (options) => {
1031
+ const resolved = options?.profile !== undefined
1032
+ ? normalizeProfile(options.profile) : storeProfile;
1033
+ assertProfileRoots(resolved, roots, collection.docPath);
1034
+ return resolved;
1035
+ };
1036
+
622
1037
  /**
623
1038
  * Resolve the profile and pushdown switches for one call.
624
1039
  * @param {any} options
@@ -626,9 +1041,8 @@ export function createQueryEngine(context) {
626
1041
  const callState = (options) => ({
627
1042
  externals: options?.externals ?? {},
628
1043
  strict: options?.strict === true,
629
- profile: options?.profile !== undefined
630
- ? normalizeProfile(options.profile)
631
- : storeProfile,
1044
+ profile: resolveProfile(options),
1045
+ profileSource: options?.profile !== undefined ? 'call' : (storeProfile === null ? null : 'store'),
632
1046
  pushdown: options?.pushdown !== false,
633
1047
  });
634
1048
 
@@ -641,6 +1055,11 @@ export function createQueryEngine(context) {
641
1055
  * synchronous driver synchronous)
642
1056
  */
643
1057
  const execute = (document, options = undefined) => {
1058
+ if (options?.strictStreaming === true) {
1059
+ throw new TypeError('strictStreaming applies to a cursor (query()); execute() answers '
1060
+ + 'the whole result by contract, so there is no stream to hold it to');
1061
+ }
1062
+ requireCallable(options, state.now);
644
1063
  const { externals, strict, profile, pushdown } = callState(options);
645
1064
  const entry = entryFor(document, strict, profile, pushdown);
646
1065
 
@@ -653,182 +1072,188 @@ export function createQueryEngine(context) {
653
1072
  // narrowing: the fetch decided nothing)
654
1073
  return chain(candidatesOf(entry, externals, diverted), (docs) => {
655
1074
  const answer = setResidualOf(entry, document)(docs, externals);
656
- countSeries(entry, 1, docs.length, itemCount(answer));
1075
+ countSeries(entry, diverted ? 1 : statementCost(entry), docs.length, itemCount(answer));
657
1076
  return answer;
658
1077
  });
659
1078
  }
660
1079
  if (entry.planned.mode === 'row') {
661
1080
  return chain(statementOf(entry), (statement) =>
662
- chain(statement.all(bindParams(entry, externals)), (rows) => {
1081
+ chain(runAll(entry, externals, statement), (rows) => {
663
1082
  const items = [];
664
1083
  for (const row of checkRowBound(entry, rows))
665
1084
  items.push(...entry.rowResidual(JSON.parse(row.doc), externals));
1085
+ countSeries(entry, statementCost(entry), rows.length, items.length);
666
1086
  return answerOf(entry, items);
667
1087
  }));
668
1088
  }
669
1089
  return chain(statementOf(entry), (statement) => {
670
1090
  if (entry.plan.aggregate !== null) {
671
- return chain(statement.get(bindParams(entry, externals)),
672
- (row) => wrapValue(entry, aggregateResult(entry, row)));
1091
+ return recoverOverflow(() => chain(runGet(entry, externals, statement), (row) => {
1092
+ const value = aggregateResult(entry, row);
1093
+ countSeries(entry, statementCost(entry), null, value === undefined ? 0 : 1);
1094
+ return wrapValue(entry, value);
1095
+ }), () => overflowResidual(entry, externals, document));
1096
+ }
1097
+ if (entry.plan.group !== null) {
1098
+ return chain(runAll(entry, externals, statement), (rows) =>
1099
+ answerOf(entry, groupItems(entry, checkRowBound(entry, rows))));
673
1100
  }
674
1101
  if (entry.plan.bucket !== null) {
675
- return chain(statement.all(bindParams(entry, externals)), (rows) => {
1102
+ return chain(runAll(entry, externals, statement), (rows) => {
676
1103
  const items = bucketItems(entry, checkRowBound(entry, rows));
677
1104
  if (items === null) return divertBucket(entry, document, externals);
678
- countSeries(entry, 1, rows.length, items.length);
1105
+ countSeries(entry, statementCost(entry), rows.length, items.length);
679
1106
  return answerOf(entry, items);
680
1107
  });
681
1108
  }
682
- return chain(statement.all(bindParams(entry, externals)), (rows) => {
683
- const docs = rowsToDocs(checkRowBound(entry, rows));
684
- countSeries(entry, 1, docs.length, docs.length);
685
- return answerOf(entry, docs);
1109
+ return chain(runAll(entry, externals, statement), (rows) => {
1110
+ checkRowBound(entry, rows);
1111
+ const items = entry.plan.project === 'document'
1112
+ ? rowsToDocs(rows)
1113
+ : 'path' in entry.plan.project
1114
+ ? rows.flatMap(projectedItems)
1115
+ : rows.flatMap((row) => projectedTreeItems(entry.plan.project, row));
1116
+ countSeries(entry, statementCost(entry), rows.length, items.length);
1117
+ return answerOf(entry, items);
686
1118
  });
687
1119
  });
688
1120
  });
689
1121
  };
690
1122
 
1123
+ /**
1124
+ * What a cursor over this call will do — one database row per pull,
1125
+ * or a buffer the first pull fills — and the construct that forces
1126
+ * the buffer. `explain()` repeats this classification for the same
1127
+ * externals, so the two can never disagree about one run.
1128
+ * @param {any} entry
1129
+ * @param {any} externals
1130
+ * @returns {{ streaming: 'row' | 'buffered',
1131
+ * barrier: import('./cursor.js').CursorBarrier | null }}
1132
+ */
1133
+ const cursorClass = (entry, externals) => {
1134
+ const buffered = (construct, reason) => ({ streaming: 'buffered', barrier: { construct, reason } });
1135
+ if (entry.planned.wrapped === true) {
1136
+ return buffered('window', BIND_REASONS.wrappedWindow);
1137
+ }
1138
+ // a plan that is a residual already buffers for its own reason,
1139
+ // whatever its externals bind to; that reason stays first
1140
+ if (entry.planned.mode === 'set' || entry.planned.mode === 'knn') {
1141
+ const forcing = entry.planned.reasons[0]
1142
+ ?? { construct: 'residual', reason: BIND_REASONS.untranslated };
1143
+ return buffered(forcing.construct, forcing.reason);
1144
+ }
1145
+ // `null` externals is the ABSTRACT question — the plan as planned,
1146
+ // every external assumed bindable — which `explain()` answers when it
1147
+ // is given no externals at all; a call always binds real ones
1148
+ const unbindable = externals === null ? null : divertingExternal(entry, externals);
1149
+ if (unbindable !== null) {
1150
+ return buffered('external', BIND_REASONS.external(unbindable, 'collection'));
1151
+ }
1152
+ if (entry.plan.bucket !== null || entry.plan.group !== null) {
1153
+ return buffered('$groupby', BIND_REASONS.bucketWhole);
1154
+ }
1155
+ return rowClassOf(connection);
1156
+ };
1157
+
691
1158
  /**
692
1159
  * A streaming cursor over the document's result ITEMS (`next()` /
693
- * `return()` plus `Symbol.asyncIterator`).
694
- * Native and row modes stream row by row; a set residual
695
- * materializes first (it is a barrier and `explain()` says so).
1160
+ * `return()` plus `Symbol.asyncIterator`), built on the one cursor
1161
+ * mechanism (cursor.js). Native and row modes pull one row per
1162
+ * `next()` from an open statement; a set residual, a k-nearest cut, a
1163
+ * diverting external, a native bucket and a chain's window
1164
+ * materialise first and say so (`streaming: 'buffered'`, with the
1165
+ * `barrier`); `signal` cancels at a row boundary.
696
1166
  * @param {any} document
697
- * @param {{ externals?: any, strict?: boolean }} [options]
1167
+ * @param {{ externals?: any, strict?: boolean, profile?: any,
1168
+ * pushdown?: boolean, signal?: AbortSignal }} [options]
698
1169
  */
699
1170
  const query = (document, options = undefined) => {
1171
+ requireCallable(options, state.now);
700
1172
  const { externals, strict, profile, pushdown } = callState(options);
701
1173
  const entry = entryFor(document, strict, profile, pushdown);
1174
+ const classified = cursorClass(entry, externals);
1175
+ refuseBuffered(options, classified, collection.docPath);
1176
+ const signal = options?.signal;
1177
+ const deadline = options?.deadline;
1178
+
1179
+ if (entry.planned.wrapped === true) {
1180
+ // a chain's element window is ONE item — the array — whatever
1181
+ // the plan mode; the cursor hands it over as `execute` answers it
1182
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1183
+ materialize: () => chain(execute(document, options), (value) => [value]) });
1184
+ }
1185
+ const diverted = mustDivert(entry, externals);
1186
+ if (diverted || entry.planned.mode === 'set' || entry.planned.mode === 'knn') {
1187
+ // the barrier: materialize candidates, pack the result items
1188
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1189
+ materialize: () => chain(guardScan(entry), () =>
1190
+ chain(candidatesOf(entry, externals, diverted), (docs) => {
1191
+ const items = packedResidualOf(entry, document)(docs, externals);
1192
+ countSeries(entry, diverted ? 1 : statementCost(entry), docs.length, items.length);
1193
+ return items;
1194
+ })) });
1195
+ }
1196
+ if (entry.plan.group !== null) {
1197
+ // a native grouping is a barrier: the groups are the answer
1198
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1199
+ materialize: () => chain(guardScan(entry), () => chain(statementOf(entry), (statement) =>
1200
+ chain(runAll(entry, externals, statement), (rows) =>
1201
+ groupItems(entry, checkRowBound(entry, rows))))) });
1202
+ }
1203
+ if (entry.plan.bucket !== null) {
1204
+ // a native bucket is a barrier: the groups are the answer
1205
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1206
+ materialize: () => chain(guardScan(entry), () => chain(statementOf(entry), (statement) =>
1207
+ chain(runAll(entry, externals, statement), (rows) => {
1208
+ const items = bucketItems(entry, checkRowBound(entry, rows));
1209
+ if (items === null) {
1210
+ return chain(divertBucket(entry, document, externals), (value) =>
1211
+ (value === undefined ? [] : Array.isArray(value) ? value : [value]));
1212
+ }
1213
+ countSeries(entry, statementCost(entry), rows.length, items.length);
1214
+ return items;
1215
+ }))) });
1216
+ }
1217
+ if (entry.plan.aggregate !== null) {
1218
+ // a native aggregate yields exactly one item; an int64 overflow
1219
+ // answers the engine's item instead
1220
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1221
+ materialize: () => chain(guardScan(entry), () => chain(statementOf(entry), (statement) =>
1222
+ recoverOverflow(() => chain(runGet(entry, externals, statement), (row) => {
1223
+ const value = aggregateResult(entry, row);
1224
+ countSeries(entry, statementCost(entry), null, value === undefined ? 0 : 1);
1225
+ return value === undefined ? [] : [value];
1226
+ }), () => chain(overflowResidual(entry, externals, document), (answer) =>
1227
+ (answer === undefined ? [] : Array.isArray(answer) ? answer : [answer]))))) });
1228
+ }
702
1229
  let pulledRows = 0;
703
-
704
- /** @type {any} */
705
- let underlying = null;
706
- /** @type {any[]} */
707
- let buffered = [];
708
- let bufferedAt = 0;
709
- let materialized = null;
710
- let done = false;
711
-
712
- const nextFromBuffer = () => ({ done: false, value: buffered[bufferedAt++] });
713
-
714
- const pull = () => {
715
- if (done) return Promise.resolve({ done: true, value: undefined });
716
- if (bufferedAt < buffered.length) return Promise.resolve(nextFromBuffer());
717
-
718
- if (entry.planned.wrapped === true) {
719
- // a chain's element window is ONE item — the array — whatever
720
- // the plan mode; the cursor hands it over as `execute` answers it
721
- if (materialized === null) {
722
- materialized = Promise.resolve(chain(execute(document, options), (value) => {
723
- buffered = [value];
724
- bufferedAt = 0;
725
- }));
726
- }
727
- return materialized.then(() => {
728
- if (bufferedAt < buffered.length) return nextFromBuffer();
729
- done = true;
730
- return { done: true, value: undefined };
731
- });
732
- }
733
- if (entry.planned.mode === 'set' || entry.planned.mode === 'knn'
734
- || mustDivert(entry, externals)) {
735
- // the barrier: materialize candidates, pack the result items
736
- if (materialized === null) {
737
- const diverted = mustDivert(entry, externals);
738
- materialized = Promise.resolve(chain(guardScan(entry), () =>
739
- chain(candidatesOf(entry, externals, diverted), (docs) => {
740
- buffered = packedResidualOf(entry, document)(docs, externals);
741
- bufferedAt = 0;
742
- countSeries(entry, 1, docs.length, buffered.length);
743
- })));
744
- }
745
- return materialized.then(() => {
746
- if (bufferedAt < buffered.length) return nextFromBuffer();
747
- done = true;
748
- return { done: true, value: undefined };
749
- });
750
- }
751
- if (entry.plan.bucket !== null) {
752
- // a native bucket is a barrier: the groups are the answer
753
- if (materialized === null) {
754
- materialized = Promise.resolve(chain(guardScan(entry), () =>
755
- chain(statementOf(entry), (statement) =>
756
- chain(statement.all(bindParams(entry, externals)), (rows) => {
757
- const items = bucketItems(entry, checkRowBound(entry, rows));
758
- if (items === null) {
759
- const answer = divertBucket(entry, document, externals);
760
- return chain(answer, (value) => {
761
- buffered = value === undefined ? []
762
- : Array.isArray(value) ? value : [value];
763
- bufferedAt = 0;
764
- });
765
- }
766
- countSeries(entry, 1, rows.length, items.length);
767
- buffered = items;
768
- bufferedAt = 0;
769
- return null;
770
- }))));
771
- }
772
- return materialized.then(() => {
773
- if (bufferedAt < buffered.length) return nextFromBuffer();
774
- done = true;
775
- return { done: true, value: undefined };
776
- });
777
- }
778
- if (entry.plan.aggregate !== null) {
779
- // a native aggregate yields exactly one item
780
- if (materialized === null) {
781
- materialized = Promise.resolve(chain(guardScan(entry), () =>
782
- chain(statementOf(entry), (statement) =>
783
- chain(statement.get(bindParams(entry, externals)), (row) => {
784
- const value = aggregateResult(entry, row);
785
- buffered = value === undefined ? [] : [value];
786
- bufferedAt = 0;
787
- }))));
788
- }
789
- return materialized.then(() => {
790
- if (bufferedAt < buffered.length) return nextFromBuffer();
791
- done = true;
792
- return { done: true, value: undefined };
793
- });
794
- }
795
-
796
- return Promise.resolve(chain(underlying === null
797
- ? chain(guardScan(entry), () => chain(statementOf(entry),
798
- (statement) => { underlying = statement.iterate(bindParams(entry, externals)); return underlying; }))
799
- : underlying, (iterator) => chain(iterator.next(), (step) => {
800
- if (step.done === true) {
801
- done = true;
802
- return { done: true, value: undefined };
803
- }
1230
+ const tally = seriesTally(entry);
1231
+ // a cursor iterates a statement of its OWN: two cursors over one
1232
+ // cached statement invalidate each other's iterator at the driver
1233
+ return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1234
+ // the bind may have to READ first (a seek's anchor), so it settles
1235
+ // before the statement it binds is prepared
1236
+ open: () => chain(guardScan(entry), () => chain(bindParams(entry, externals),
1237
+ (params) => chain(connection.prepare(entry.sql, { readOnly: true, ephemeral: true }),
1238
+ (statement) => statement.iterate(params)))),
1239
+ items: (row) => {
804
1240
  pulledRows++;
805
1241
  if (entry.rowBound !== null && pulledRows > entry.rowBound) {
806
- done = true;
807
- if (typeof iterator.return === 'function') iterator.return(undefined);
808
1242
  throw new DbRuntimeError('JD2007',
809
1243
  `the fetch crossed the profile's maxRows bound of ${entry.rowBound}`,
810
1244
  { docPath: collection.docPath, collection: collection.name });
811
1245
  }
812
- const doc = JSON.parse(step.value.doc);
813
- if (entry.rowResidual === null) return { done: false, value: doc };
814
- buffered = entry.rowResidual(doc, externals);
815
- bufferedAt = 0;
816
- return bufferedAt < buffered.length ? nextFromBuffer() : pull();
817
- })));
818
- };
819
-
820
- const close = () => {
821
- done = true;
822
- if (underlying !== null && typeof underlying.return === 'function')
823
- underlying.return(undefined);
824
- return Promise.resolve({ done: true, value: undefined });
825
- };
826
-
827
- return {
828
- next: () => pull(),
829
- return: () => close(),
830
- [Symbol.asyncIterator]() { return this; },
831
- };
1246
+ checkByteBound(entry, row);
1247
+ const items = entry.plan.project === 'document'
1248
+ ? (entry.rowResidual === null ? [JSON.parse(row.doc)]
1249
+ : entry.rowResidual(JSON.parse(row.doc), externals))
1250
+ : 'path' in entry.plan.project
1251
+ ? projectedItems(row)
1252
+ : projectedTreeItems(entry.plan.project, row);
1253
+ tally?.row(items.length);
1254
+ return items;
1255
+ },
1256
+ onSettle: tally === null ? undefined : tally.settle });
832
1257
  };
833
1258
 
834
1259
  /**
@@ -842,8 +1267,19 @@ export function createQueryEngine(context) {
842
1267
  * @param {{ externals?: any, strict?: boolean }} [options]
843
1268
  */
844
1269
  const explain = (document, options = undefined) => {
845
- const { externals, strict, profile, pushdown } = callState(options);
1270
+ const { externals, strict, profile, profileSource, pushdown } = callState(options);
846
1271
  const entry = entryFor(document, strict, profile, pushdown);
1272
+ // the run this call would make: bound against the externals it was
1273
+ // given, a diversion reads the whole collection through the
1274
+ // diversion statement and answers in the set residual — so that is
1275
+ // the mode, the SQL and the barrier reported, not the native plan's
1276
+ // no externals at all is the abstract plan, as it always was; given
1277
+ // externals — even a partial set — are bound as execute would bind them
1278
+ const bound = options?.externals !== undefined;
1279
+ const classified = cursorClass(entry, bound ? externals : null);
1280
+ const diverted = bound && mustDivert(entry, externals);
1281
+ const chosen = diverted ? fullScanEmitted(entry) : entry;
1282
+ const mode = diverted ? 'set' : entry.planned.mode;
847
1283
 
848
1284
  const touchedColumns = new Set();
849
1285
  // an R*Tree probe touches no generated column at all — the index it
@@ -872,6 +1308,14 @@ export function createQueryEngine(context) {
872
1308
  if (aggregate.ref?.column) touchedColumns.add(aggregate.ref.column);
873
1309
  }
874
1310
  }
1311
+ if (entry.plan.group !== null) {
1312
+ for (const key of entry.plan.group.keys) {
1313
+ if (key.ref.column) touchedColumns.add(key.ref.column);
1314
+ }
1315
+ for (const aggregate of entry.plan.group.aggregates) {
1316
+ if (aggregate.ref?.column) touchedColumns.add(aggregate.ref.column);
1317
+ }
1318
+ }
875
1319
  const indexes = [
876
1320
  ...physicalPlan.expected.indexes
877
1321
  .filter((index) => index.columns.some((column) => touchedColumns.has(column)))
@@ -879,20 +1323,68 @@ export function createQueryEngine(context) {
879
1323
  ...[...touchedVirtual].sort(),
880
1324
  ];
881
1325
 
882
- const params = entry.slots.map((slot) => {
1326
+ const params = chosen.slots.map((slot) => {
883
1327
  if ('external' in slot) return { external: slot.external };
884
1328
  if ('derived' in slot) return { derived: { ...slot.derived } };
1329
+ if ('typed' in slot) return { typed: { ...slot.typed } };
885
1330
  return { literal: slot.literal };
886
1331
  });
887
- const eqpParams = entry.slots.map((slot) => {
1332
+ // EXPLAIN reads the statement's SHAPE: a seek's anchor is a value
1333
+ // the database would answer for a run, and explaining a plan runs
1334
+ // nothing, so the slot it fills is described and left unbound
1335
+ const eqpParams = chosen.slots.map((slot) => {
1336
+ if ('typed' in slot) return null;
888
1337
  const value = slotValue(slot, externals);
889
1338
  return bindable(value) ? value : null;
890
1339
  });
1340
+ // a diversion is one more reason the engine answers, appended after
1341
+ // the plan's own: a residual stays a residual for its own reason,
1342
+ // and a native plan's only reason is the value that would not bind
1343
+ const reasons = diverted
1344
+ ? [...entry.planned.reasons, { construct: 'external',
1345
+ reason: BIND_REASONS.external(divertingExternal(entry, externals), 'collection') }]
1346
+ : entry.planned.reasons;
891
1347
 
892
1348
  const rank = entry.plan.rank;
893
- return chain(connection.prepare(dialect.explainQuery(entry.sql)), (statement) =>
1349
+ return chain(connection.prepare(dialect.explainQuery(chosen.sql), { readOnly: true }), (statement) =>
894
1350
  chain(statement.all(eqpParams), (rows) => ({
895
- mode: entry.planned.mode,
1351
+ mode,
1352
+ // what a cursor over this call does — one row per pull, or a
1353
+ // buffer — and the construct that forces the buffer: the same
1354
+ // classification the cursor itself carries, so the two agree
1355
+ streaming: classified.streaming,
1356
+ barrier: classified.barrier,
1357
+ // the profile that applied and every bound it imposed (D7)
1358
+ budget: budgetOf(profile, profileSource, connection.capabilities),
1359
+ // the order the STATEMENT executes under — the plan's declared
1360
+ // terms and the tie-breaker the emitter appends, in the same
1361
+ // normalized form the emitter renders (D6: read from the plan,
1362
+ // never parsed back out of SQL). `null` is the honest answer for
1363
+ // a statement that orders nothing at all
1364
+ order: planOrder(diverted ? entry.fullScanShape() : entry.plan),
1365
+ // what the statement projects: `null` when it reads the whole
1366
+ // document, one `path` when it projects a single member, and
1367
+ // `paths` — one per DISTINCT leaf, in fetch order — when it
1368
+ // projects a nested shape the decoder rebuilds
1369
+ projection: entry.plan.project === 'document' ? null
1370
+ : 'path' in entry.plan.project
1371
+ ? { path: segmentsOf(entry.plan.project.path) }
1372
+ : { paths: entry.plan.project.leaves.map(segmentsOf) },
1373
+ // the projection that stayed behind, when one did: the one-row
1374
+ // document the row residual runs per fetched row
1375
+ residualProjection: entry.planned.rowReturn?.$return?.[0] ?? null,
1376
+ // the GROUPING the statement performs, when it performs one:
1377
+ // the key names with the member each reads, the aggregates with
1378
+ // the function and the member each folds, and how the groups
1379
+ // come out. `null` when nothing is grouped natively
1380
+ group: entry.plan.group === null ? null : {
1381
+ keys: entry.plan.group.keys.map((key) => ({ as: key.as, path: segmentsOf(key.ref) })),
1382
+ aggregates: entry.plan.group.aggregates.map((entry2) => ({
1383
+ fn: entry2.fn, path: entry2.ref === null ? null : segmentsOf(entry2.ref) })),
1384
+ order: entry.plan.group.order === 'first-seen' ? 'first-seen'
1385
+ : entry.plan.group.order.map((term) => ({
1386
+ key: entry.plan.group.keys[term.index].as, desc: term.desc })),
1387
+ },
896
1388
  // a chain's element window (`[<phrase>]`): the phrase planned as
897
1389
  // if bare, its rows answered as the one array item
898
1390
  wrapped: entry.planned.wrapped === true,
@@ -901,7 +1393,7 @@ export function createQueryEngine(context) {
901
1393
  functions: [...entry.dependencies.functions],
902
1394
  collations: [...entry.dependencies.collations],
903
1395
  limits: entry.residualLimits ?? entry.limits,
904
- sql: entry.sql,
1396
+ sql: chosen.sql,
905
1397
  params,
906
1398
  indexes,
907
1399
  prefilters: entry.planned.prefilters.map((prefilter) => ({ ...prefilter,
@@ -910,21 +1402,34 @@ export function createQueryEngine(context) {
910
1402
  // reads, the window the cut serves, the margin it keeps, and
911
1403
  // who decides the order — always the engine
912
1404
  rank: rank === null ? null : {
913
- column: rank.column,
914
- dims: rank.dims,
1405
+ // every declared width the plan can bind, and — when the call
1406
+ // was given its externals — the one it selected. The probe is
1407
+ // named, never printed
1408
+ alternatives: entry.rankAlternatives.map((alternative) =>
1409
+ ({ column: alternative.column, dims: alternative.dims })),
1410
+ selected: !bound || 'lit' in rank.probe
1411
+ ? (('lit' in rank.probe) ? rank.alternatives[0].dims : null)
1412
+ : rankAlternativeFor(entry, externals[rank.probe.ext])?.dims ?? null,
915
1413
  probe: 'lit' in rank.probe ? { literal: [...rank.probe.lit] } : { external: rank.probe.ext },
916
1414
  limit: rank.limit,
917
1415
  offset: rank.offset,
918
1416
  margin: rank.margin,
919
1417
  decides: 'engine',
920
1418
  },
921
- residual: entry.planned.mode === 'native'
1419
+ residual: mode === 'native'
922
1420
  ? null
923
- : { mode: entry.planned.mode, reasons: entry.planned.reasons },
924
- barriers: entry.planned.mode === 'set' || entry.planned.mode === 'knn'
925
- ? entry.planned.reasons.map((r) => ({ operator: r.construct, reason: r.reason }))
1421
+ : { mode, reasons },
1422
+ barriers: mode === 'set' || mode === 'knn'
1423
+ ? reasons.map((r) => ({ operator: r.construct, reason: r.reason }))
926
1424
  : [],
927
1425
  udfs: [...entry.planned.udfs],
1426
+ // a pushed aggregate that overflowed int64 at run time: the
1427
+ // engine answered the document instead, and says so here
1428
+ fallback: entry.overflowRuns === undefined ? null : {
1429
+ construct: 'overflow',
1430
+ runs: entry.overflowRuns,
1431
+ reason: BIND_REASONS.overflow,
1432
+ },
928
1433
  // the temporal record: what the document asked, which declared
929
1434
  // index the fetch seeks through, and which kernel finished it.
930
1435
  // The counts are the LAST ACTUAL execution's — `null` before
@@ -934,21 +1439,107 @@ export function createQueryEngine(context) {
934
1439
  ...entry.planned.series,
935
1440
  counts: entry.seriesCounts === null ? null : { ...entry.seriesCounts },
936
1441
  },
937
- scanNarrative: rows.map((row) => String(row.detail)).join('; '),
1442
+ scanNarrative: dialect.explainLines(rows).join('; '),
938
1443
  })));
939
1444
  };
940
1445
 
941
- return { execute, query, explain, shape,
942
- stats: () => ({ knn: { ...knnStats }, series: { ...seriesStats } }) };
1446
+ return { execute: bounded(execute, driverWrap), query, explain: bounded(explain, driverWrap), shape,
1447
+ stats: () => ({ knn: { ...knnStats }, series: { ...seriesStats }, bind: { ...bindStats } }) };
1448
+ }
1449
+
1450
+ /**
1451
+ * `strictStreaming` (D6 applied to memory): a plan that would buffer is
1452
+ * declined by name before any statement runs, never run with its memory
1453
+ * behaviour quietly changed.
1454
+ * @param {any} options
1455
+ * @param {{ streaming: string, barrier: import('./cursor.js').CursorBarrier | null }} classified
1456
+ * @param {string | undefined} docPath
1457
+ */
1458
+ function refuseBuffered(options, classified, docPath) {
1459
+ if (options?.strictStreaming !== true || classified.streaming !== 'buffered') return;
1460
+ const barrier = classified.barrier;
1461
+ throw new DbCompileError('JD0037',
1462
+ `strictStreaming refused a plan that buffers: '${barrier?.construct}' — ${barrier?.reason}`,
1463
+ docPath);
943
1464
  }
944
1465
 
945
1466
  // ————— The entity query surface (the second document kind) —————
946
1467
 
947
1468
  import { mergeEntityRow, parseGraphRow } from './graph.js';
948
- import { relationTables } from './model.js';
1469
+ import { relationTables, joinTableRoots } from './model.js';
949
1470
 
950
1471
  /** The default include depth bound (D14: printed, never silent). */
951
1472
  export const INCLUDE_DEPTH_DEFAULT = 3;
1473
+ /** The default per-root bounds of an included to-many relation
1474
+ * (MODEL-FORMAT §10.4): rows per parent, and serialised bytes per
1475
+ * parent. A bound always exists — one root that aggregates an unbounded
1476
+ * relation is not a bounded item — and the unbounded case is spelled
1477
+ * (`maxRows: Infinity`), never inherited. An explicit `take` is the row
1478
+ * bound of the include it windows. */
1479
+ export const INCLUDE_ROWS_DEFAULT = 1000;
1480
+ export const INCLUDE_BYTES_DEFAULT = 1_048_576;
1481
+
1482
+ /**
1483
+ * Plan one `where` EXPRESSION over `$it` against one entity: the plan
1484
+ * predicate when every conjunct translates, else the first refusal —
1485
+ * the one translation the include tree, the profile's mandatory
1486
+ * predicates and the entity residual's narrowing all share.
1487
+ * @param {any} expression
1488
+ * @param {any} entity - the normalized entity
1489
+ * @param {any} entityMapping - `explainMapping(...).entities[name]`
1490
+ * @param {any} analyzeOpts
1491
+ * @returns {{ filter: any } | { refusal: { construct: string, reason: string } } | { error: Error }}
1492
+ */
1493
+ function planEntityWhere(expression, entity, entityMapping, analyzeOpts) {
1494
+ const wrapper = { $for: { it: '$[*]' }, $where: expression, $return: '$it' };
1495
+ let analysis;
1496
+ try {
1497
+ analysis = analyzeQuery(wrapper, analyzeOpts);
1498
+ }
1499
+ catch (cause) {
1500
+ return { error: /** @type {Error} */ (cause) };
1501
+ }
1502
+ const flwor = analysis.root;
1503
+ const slot = flwor.forBindings[0].slot;
1504
+ const shape = entityShape(entity, entityMapping);
1505
+ const conjuncts = flwor.where.kind === 'op' && flwor.where.name === '$and'
1506
+ ? flwor.where.args : [flwor.where];
1507
+ let filter = null;
1508
+ for (const conjunct of conjuncts) {
1509
+ const outcome = planEntityPredicate(conjunct, slot, shape);
1510
+ if ('refusal' in outcome) return { refusal: outcome.refusal };
1511
+ filter = conjoin(filter, outcome.pred);
1512
+ }
1513
+ return { filter };
1514
+ }
1515
+
1516
+ /**
1517
+ * The profile's mandatory predicate for one entity, translated — a
1518
+ * host-configured predicate that cannot translate is a host programming
1519
+ * error (`TypeError`), exactly as on a collection: there is no residual
1520
+ * to hide it in, because the point is that it binds the fetch.
1521
+ * @param {any} profile
1522
+ * @param {any} entity
1523
+ * @param {any} entityMapping
1524
+ * @param {any} analyzeOpts
1525
+ * @returns {any} a plan predicate, or null
1526
+ */
1527
+ function mandatoryEntityPredicate(profile, entity, entityMapping, analyzeOpts) {
1528
+ const expression = profile?.predicates?.[entity.name];
1529
+ if (expression === undefined) return null;
1530
+ const planned = planEntityWhere(expression, entity, entityMapping, analyzeOpts);
1531
+ if ('filter' in planned && planned.filter !== null) return planned.filter;
1532
+ throw new TypeError(`a profile predicate must translate natively (it binds the database-side fetch of '${
1533
+ entity.name}'); this one refused: ${'refusal' in planned ? planned.refusal.reason
1534
+ : 'error' in planned ? planned.error.message : 'it selects nothing'}`);
1535
+ }
1536
+
1537
+ /**
1538
+ * The profile's compile-time refusal for an entity document (`JD0011`).
1539
+ * @param {string} reason
1540
+ * @param {string | undefined} docPath
1541
+ */
1542
+ const profileEntityRefusal = (reason, docPath) => new DbCompileError('JD0011', reason, docPath);
952
1543
 
953
1544
  /**
954
1545
  * The store-level entity query engine: documents over the
@@ -961,19 +1552,55 @@ export const INCLUDE_DEPTH_DEFAULT = 3;
961
1552
  * @returns {any}
962
1553
  */
963
1554
  export function createEntityQueryEngine(context) {
964
- const { connection, entities, mapping, state } = context;
1555
+ const { connection, state } = context;
1556
+ // the query roots: the model's entities, plus the read-only
1557
+ // pseudo-entity each declared join table contributes (§10.7). They
1558
+ // are visible to the PLANNER and to `$.<Name>[*]`, and to nothing
1559
+ // that writes — `store.entity(name)` reads the model's own map
1560
+ const joinRoots = joinTableRoots(context.entities, context.mapping);
1561
+ const entities = joinRoots.entities.size === 0
1562
+ ? context.entities
1563
+ : new Map([...context.entities, ...joinRoots.entities]);
1564
+ const mapping = joinRoots.entities.size === 0
1565
+ ? context.mapping
1566
+ : { ...context.mapping,
1567
+ entities: { ...context.mapping.entities, ...joinRoots.mappings } };
1568
+ const storeProfile = context.profile ?? null;
1569
+ const roots = context.roots ?? [...entities.keys()];
965
1570
  const operators = state.operators ?? null;
1571
+ const analyzeOpts = operators ?? undefined;
966
1572
  const zoneProvider = state.zoneProvider ?? null;
1573
+ /** Every driver failure this engine meets, classified under the
1574
+ * entity root. */
1575
+ const driverWrap = (/** @type {any} */ error) => wrapDriverError(error, { docPath: '/entities' });
967
1576
  const dialect = connection.dialect;
968
1577
  const q = dialect.quoteIdentifier;
969
- const physicalOf = (name) => ({ table: mapping.entities[name].table });
1578
+ const physicalOf = (name) => ({ table: mapping.entities[name].table,
1579
+ // a join-table root has no document column of its own (§10.7)
1580
+ document: mapping.entities[name].document !== false });
970
1581
  // the relation tables of every root this engine serves (§10.1): the
971
1582
  // engine is the scope every entity set of the store shares, so a
972
1583
  // producer holding one set can follow a hop into another root
973
1584
  const relations = relationTables(entities);
974
1585
 
975
- const entryFor = (document, pushdown) => {
976
- const key = ['E', document, dialect.name, pushdown];
1586
+ /** The switches one call resolves: the profile per call replaces the
1587
+ * store's, normalized over the safe defaults, as on a collection. */
1588
+ const callState = (options) => {
1589
+ const profile = options?.profile !== undefined
1590
+ ? normalizeProfile(options.profile) : storeProfile;
1591
+ assertProfileRoots(profile, roots, '/entities');
1592
+ return {
1593
+ externals: options?.externals ?? {},
1594
+ strict: options?.strict === true,
1595
+ pushdown: options?.pushdown !== false,
1596
+ profile,
1597
+ profileSource: options?.profile !== undefined ? 'call'
1598
+ : (storeProfile === null ? null : 'store'),
1599
+ };
1600
+ };
1601
+
1602
+ const entryFor = (document, pushdown, profile = null) => {
1603
+ const key = ['E', document, dialect.name, pushdown, profile];
977
1604
  const cached = state.cache.get(key);
978
1605
  if (cached !== undefined) {
979
1606
  state.counters.hits++;
@@ -991,7 +1618,51 @@ export function createEntityQueryEngine(context) {
991
1618
  }
992
1619
  if (!pushdown) {
993
1620
  planned = { ...planned, mode: 'set', plan: null,
994
- reasons: [{ construct: 'pushdown', reason: 'disabled by the harness switch' }] };
1621
+ reasons: [{ construct: 'pushdown', reason: BIND_REASONS.pushdown }] };
1622
+ }
1623
+ const docPath = entities.get(planned.referenced[0])?.docPath;
1624
+ // the profile, applied exactly as on a collection (MODEL-FORMAT §8):
1625
+ // the roots a document may read, the references it may make, the
1626
+ // predicate every fetch of a root must wear, the row bound every
1627
+ // fetch carries, and the shape refusal of a whole-root residual
1628
+ if (profile !== null) {
1629
+ for (const name of planned.referenced) {
1630
+ if (profile.collections !== null && !profile.collections.includes(name))
1631
+ throw profileEntityRefusal(`the profile does not allow querying entity '${name}'`, docPath);
1632
+ // the member allow-list, per referenced root: every member path
1633
+ // the document reads on a binding over that entity's array
1634
+ const denied = memberDenial(profile, name,
1635
+ planned.analysis.root, isEntityRootSource(name));
1636
+ if (denied !== null) throw profileEntityRefusal(denied, entities.get(name)?.docPath);
1637
+ }
1638
+ const deps = planned.analysis.dependencies;
1639
+ for (const name of deps.functions) {
1640
+ if (!profile.functions.includes(name))
1641
+ throw profileEntityRefusal(`the profile does not allow the host function '${name}'`, docPath);
1642
+ }
1643
+ for (const name of deps.collations) {
1644
+ if (!profile.collations.includes(name))
1645
+ throw profileEntityRefusal(`the profile does not allow the collation '${name}'`, docPath);
1646
+ }
1647
+ for (const external of planned.analysis.externals) {
1648
+ if (!profile.externals.includes(external.name))
1649
+ throw profileEntityRefusal(`the profile does not declare the external '${external.name}'`, docPath);
1650
+ }
1651
+ if (planned.mode !== 'native' && profile.refuseFullScan === true) {
1652
+ // the residual reads every row of every referenced root before
1653
+ // the engine decides — a full-table scan by shape, refused at
1654
+ // preflight rather than estimated (D6)
1655
+ throw profileEntityRefusal('the profile refuses a full-table scan, and the residual this '
1656
+ + `document needs fetches every row of ${planned.referenced.join(', ')} `
1657
+ + `('${planned.reasons[0]?.construct}' — ${planned.reasons[0]?.reason})`, docPath);
1658
+ }
1659
+ }
1660
+ const mandatory = new Map();
1661
+ if (profile !== null) {
1662
+ for (const name of planned.referenced) {
1663
+ const predicate = mandatoryEntityPredicate(profile, entities.get(name), mapping.entities[name], analyzeOpts);
1664
+ if (predicate !== null) mandatory.set(name, predicate);
1665
+ }
995
1666
  }
996
1667
  const entry = {
997
1668
  planned,
@@ -999,10 +1670,30 @@ export function createEntityQueryEngine(context) {
999
1670
  slots: null,
1000
1671
  statement: null,
1001
1672
  setResidual: null,
1673
+ packedResidual: null,
1002
1674
  fetchers: null,
1675
+ mandatory,
1676
+ rowBound: profile === null ? null : profile.maxRows,
1677
+ byteBound: profile === null ? null : profile.maxBytes,
1678
+ residualLimits: profile === null ? undefined : profile.limits,
1679
+ needsScanCheck: profile !== null && profile.refuseFullScan === true,
1680
+ scanChecked: false,
1003
1681
  };
1004
1682
  if (planned.mode === 'native') {
1005
- const emitted = emitEntityPlan(planned.plan, dialect, physicalOf);
1683
+ let plan = planned.plan;
1684
+ if (mandatory.size > 0) {
1685
+ plan = { ...plan, filters: plan.filters.map((entry) => {
1686
+ const binding = plan.bindings.find((candidate) => candidate.name === entry.binding);
1687
+ const predicate = mandatory.get(binding.entity);
1688
+ return predicate === undefined ? entry : { ...entry, filter: conjoin(entry.filter, predicate) };
1689
+ }) };
1690
+ }
1691
+ if (entry.rowBound !== null && plan.aggregate === null) {
1692
+ const cap = entry.rowBound + 1;
1693
+ plan = { ...plan, window: plan.window === null ? { offset: 0, limit: cap }
1694
+ : { offset: plan.window.offset, limit: plan.window.limit === null ? cap : Math.min(plan.window.limit, cap) } };
1695
+ }
1696
+ const emitted = emitEntityPlan(plan, dialect, physicalOf);
1006
1697
  entry.sql = emitted.sql;
1007
1698
  entry.slots = emitted.slots;
1008
1699
  }
@@ -1012,27 +1703,90 @@ export function createEntityQueryEngine(context) {
1012
1703
  return entry;
1013
1704
  };
1014
1705
 
1015
- /** Fetch every referenced entity's rows and build the in-memory root. */
1706
+ /** Refuse a native fetch that crossed the profile's row bound (JD2007). */
1707
+ const checkRows = (entry, rows, name) => {
1708
+ if (entry.rowBound !== null && rows.length > entry.rowBound) {
1709
+ throw new DbRuntimeError('JD2007',
1710
+ `the fetch crossed the profile's maxRows bound of ${entry.rowBound}`,
1711
+ { docPath: entities.get(name)?.docPath, collection: name });
1712
+ }
1713
+ return rows;
1714
+ };
1715
+ /** One merged entity document against the profile's byte bound
1716
+ * (JD2076): measured after the merge, because the mapped scalars live
1717
+ * in columns and the row's own JSON text holds only the rest. */
1718
+ const checkBytes = (entry, doc, name) => {
1719
+ if (entry.byteBound === null) return doc;
1720
+ const bytes = utf8Length(JSON.stringify(doc));
1721
+ if (bytes > entry.byteBound) {
1722
+ throw new DbRuntimeError('JD2076',
1723
+ `an item of ${bytes} serialised bytes exceeds the profile's maxBytes bound of ${entry.byteBound}`,
1724
+ { docPath: entities.get(name)?.docPath, collection: name });
1725
+ }
1726
+ return doc;
1727
+ };
1728
+ /** The plan-shape refusal on a native plan: a full-table SCAN of any
1729
+ * referenced root, verified against the database's own plan output. */
1730
+ const guardEntityScan = (entry) => {
1731
+ if (!entry.needsScanCheck || entry.scanChecked) return null;
1732
+ const eqpParams = entry.slots.map((slot) => ('literal' in slot ? slot.literal : null));
1733
+ return chain(connection.prepare(dialect.explainQuery(entry.sql), { readOnly: true }), (statement) =>
1734
+ chain(statement.all(eqpParams), (rows) => {
1735
+ // the entity statement aliases its tables `t0`, `t1`, … and the
1736
+ // database's narrative names the alias; a bare table name is
1737
+ // the residual fetcher's spelling
1738
+ const tables = entry.planned.referenced.map((name) => mapping.entities[name].table);
1739
+ const lines = dialect.explainLines(rows);
1740
+ if (lines.some((line) => dialect.isFullScan(line, tables))) {
1741
+ throw profileEntityRefusal('the profile refuses a full-table scan of '
1742
+ + `${entry.planned.referenced.join(', ')} (${lines.join('; ')})`,
1743
+ entities.get(entry.planned.referenced[0])?.docPath);
1744
+ }
1745
+ entry.scanChecked = true;
1746
+ return null;
1747
+ }));
1748
+ };
1749
+
1750
+ /** Fetch every referenced entity's rows and build the in-memory
1751
+ * root — each fetch wearing the profile's mandatory predicate for its
1752
+ * entity and its row bound (`LIMIT maxRows + 1`, refused when crossed),
1753
+ * so a residual's input is as bounded as a native answer. */
1016
1754
  const fetchRoot = (entry) => {
1017
1755
  if (entry.fetchers === null) {
1018
1756
  entry.fetchers = [...(entry.planned.referenced.length === 0
1019
- ? entities.keys() : entry.planned.referenced)].map((name) => ({
1020
- name,
1021
- sql: `SELECT ${q('t')}.*, ${dialect.jsonText(`${q('t')}.${q('doc')}`)} AS ${q('__doc')} `
1022
- + `FROM ${q(mapping.entities[name].table)} AS ${q('t')} ORDER BY ${q('t')}.${dialect.rowIdentity()}`,
1023
- statement: null,
1024
- }));
1757
+ ? entities.keys() : entry.planned.referenced)].map((name) => {
1758
+ /** @type {any[]} */
1759
+ const slots = [];
1760
+ const emitters = createEntityPredicateEmitters(dialect, (slot) => {
1761
+ slots.push(slot);
1762
+ return dialect.parameterRef(slots.length, 'v');
1763
+ });
1764
+ const predicate = entry.mandatory.get(name);
1765
+ const where = predicate === undefined ? ''
1766
+ : ` WHERE ${emitters.emitPred(q('t'), `${q('t')}.${q('doc')}`, predicate)}`;
1767
+ const limit = entry.rowBound === null ? '' : ` ${dialect.limitClause(entry.rowBound + 1, undefined)}`;
1768
+ return {
1769
+ name,
1770
+ sql: `SELECT ${q('t')}.*, ${mapping.entities[name].document === false
1771
+ ? dialect.stringLiteral('{}')
1772
+ : dialect.jsonText(`${q('t')}.${q('doc')}`)} AS ${q('__doc')} `
1773
+ + `FROM ${q(mapping.entities[name].table)} AS ${q('t')}${where} `
1774
+ + `ORDER BY ${q('t')}.${dialect.rowIdentity()}${limit}`,
1775
+ params: slots.map((slot) => slot.literal),
1776
+ statement: null,
1777
+ };
1778
+ });
1025
1779
  }
1026
1780
  /** @type {any} */
1027
1781
  const root = {};
1028
1782
  const next = (i) => {
1029
1783
  if (i >= entry.fetchers.length) return root;
1030
1784
  const fetcher = entry.fetchers[i];
1031
- if (fetcher.statement === null) fetcher.statement = connection.prepare(fetcher.sql);
1785
+ if (fetcher.statement === null) fetcher.statement = connection.prepare(fetcher.sql, { readOnly: true });
1032
1786
  return chain(fetcher.statement, (statement) =>
1033
- chain(statement.all([]), (rows) => {
1034
- root[fetcher.name] = rows.map((row) =>
1035
- mergeEntityRow(mapping.entities[fetcher.name], row, '__doc'));
1787
+ chain(statement.all(fetcher.params), (rows) => {
1788
+ root[fetcher.name] = checkRows(entry, rows, fetcher.name).map((row) =>
1789
+ checkBytes(entry, mergeEntityRow(mapping.entities[fetcher.name], row, '__doc'), fetcher.name));
1036
1790
  return next(i + 1);
1037
1791
  }));
1038
1792
  };
@@ -1041,16 +1795,16 @@ export function createEntityQueryEngine(context) {
1041
1795
 
1042
1796
  const runResidual = (entry, document, externals) => {
1043
1797
  if (entry.setResidual === null)
1044
- entry.setResidual = compileSetResidual(document, undefined, operators, zoneProvider);
1798
+ entry.setResidual = compileSetResidual(document, entry.residualLimits, operators, zoneProvider);
1045
1799
  return chain(fetchRoot(entry), (root) => entry.setResidual(root, externals));
1046
1800
  };
1047
1801
 
1048
1802
  const execute = (document, options = undefined) => {
1049
- const externals = options?.externals ?? {};
1050
- const pushdown = options?.pushdown !== false;
1051
- const entry = entryFor(document, pushdown);
1803
+ requireCallable(options, state.now);
1804
+ const { externals, strict, pushdown, profile } = callState(options);
1805
+ const entry = entryFor(document, pushdown, profile);
1052
1806
  if (entry.planned.mode !== 'native') {
1053
- if (options?.strict === true) {
1807
+ if (strict) {
1054
1808
  const forcing = entry.planned.reasons[0];
1055
1809
  throw new DbCompileError('JD0010',
1056
1810
  `strict mode refused a residual: '${forcing.construct}' — ${forcing.reason}`);
@@ -1065,43 +1819,200 @@ export function createEntityQueryEngine(context) {
1065
1819
  const params = entry.slots.map((slot) => slotValue(slot, externals));
1066
1820
  if (params.some((value) => !bindable(value)))
1067
1821
  return runResidual(entry, document, externals);
1068
- if (entry.statement === null) entry.statement = connection.prepare(entry.sql);
1069
- return chain(entry.statement, (statement) => {
1822
+ if (entry.statement === null) entry.statement = connection.prepare(entry.sql, { readOnly: true });
1823
+ return chain(guardEntityScan(entry), () => chain(entry.statement, (statement) => {
1070
1824
  if (entry.planned.plan.aggregate === 'count')
1071
1825
  return chain(statement.get(params), (row) => wrapValue(entry, row?.value ?? 0));
1072
1826
  return chain(statement.all(params), (rows) => {
1827
+ const project = entry.planned.plan.project;
1828
+ if (project != null) {
1829
+ return answerOf(entry,
1830
+ rows.flatMap((row) => projectedTreeItems(project, row, 'p')));
1831
+ }
1073
1832
  const retEntity = entry.planned.plan.bindings
1074
1833
  .find((binding) => binding.name === entry.planned.plan.ret).entity;
1075
- return answerOf(entry, rows.map((row) =>
1076
- mergeEntityRow(mapping.entities[retEntity], row, '__doc')));
1834
+ return answerOf(entry, checkRows(entry, rows, retEntity).map((row) =>
1835
+ checkBytes(entry, mergeEntityRow(mapping.entities[retEntity], row, '__doc'), retEntity)));
1077
1836
  });
1078
- });
1837
+ }));
1838
+ };
1839
+
1840
+ /** The item-packing residual for a cursor over the fetched root. */
1841
+ const packedResidualOf = (entry, document) => {
1842
+ if (entry.packedResidual === null)
1843
+ entry.packedResidual = compilePackedResidual(document, entry.residualLimits, operators, zoneProvider);
1844
+ return entry.packedResidual;
1845
+ };
1846
+
1847
+ /**
1848
+ * The bind-time diversion, named: the first parameter slot whose
1849
+ * value the database cannot take — a missing external, a boolean, a
1850
+ * null, a region with no box — or `null` when the statement binds.
1851
+ * @param {any} entry
1852
+ * @param {any} externals
1853
+ * @returns {{ construct: string, reason: string } | null}
1854
+ */
1855
+ const divertReason = (entry, externals) => {
1856
+ for (const slot of entry.slots) {
1857
+ if (bindable(slotValue(slot, externals))) continue;
1858
+ const name = 'external' in slot ? slot.external
1859
+ : 'derived' in slot ? slot.derived.external : null;
1860
+ return { construct: 'external', reason: BIND_REASONS.external(name, 'root') };
1861
+ }
1862
+ return null;
1863
+ };
1864
+
1865
+ /**
1866
+ * What a cursor over this call will do, and why — the collection
1867
+ * engine's classification over the entity plan shapes: a chain's
1868
+ * window, a set residual (its first reason names the construct), a
1869
+ * diverting external, else one row per pull.
1870
+ * @param {any} entry
1871
+ * @param {any} externals
1872
+ * @returns {{ streaming: 'row' | 'buffered',
1873
+ * barrier: import('./cursor.js').CursorBarrier | null }}
1874
+ */
1875
+ const cursorClass = (entry, externals) => {
1876
+ const buffered = (barrier) => ({ streaming: 'buffered', barrier });
1877
+ if (entry.planned.wrapped === true) {
1878
+ return buffered({ construct: 'window', reason: BIND_REASONS.wrappedWindow });
1879
+ }
1880
+ if (entry.planned.mode !== 'native') {
1881
+ const forcing = entry.planned.reasons[0]
1882
+ ?? { construct: 'residual', reason: BIND_REASONS.untranslated };
1883
+ return buffered({ construct: forcing.construct, reason: forcing.reason });
1884
+ }
1885
+ // `null` externals is the abstract question: the plan as planned
1886
+ const diverted = externals === null ? null : divertReason(entry, externals);
1887
+ return diverted === null ? rowClassOf(connection) : buffered(diverted);
1888
+ };
1889
+
1890
+ /**
1891
+ * The item cursor over an entity document — the collection engine's
1892
+ * `query()` over the second document kind, on the one cursor
1893
+ * mechanism (cursor.js): a native selection or join pulls one row
1894
+ * per `next()` from an open statement and merges it into its entity
1895
+ * document; a count yields its one item; a set residual and a
1896
+ * diverting external materialise the fetched root first and say so.
1897
+ * `register`, when given, is the unit of work's registration: every
1898
+ * yielded entity document passes through it, which is why the
1899
+ * document must return a bare entity binding (`JD0034` otherwise) —
1900
+ * a projection is not a snapshot anything could save.
1901
+ * @param {any} document
1902
+ * @param {{ externals?: any, strict?: boolean, pushdown?: boolean,
1903
+ * signal?: AbortSignal }} [options]
1904
+ * @param {((entity: string, doc: any) => any) | undefined} [register]
1905
+ */
1906
+ const query = (document, options = undefined, register = undefined, cursorFactory = createCursor) => {
1907
+ requireCallable(options, state.now);
1908
+ const { externals, strict, pushdown, profile } = callState(options);
1909
+ const entry = entryFor(document, pushdown, profile);
1910
+ if (strict && entry.planned.mode !== 'native') {
1911
+ const forcing = entry.planned.reasons[0];
1912
+ throw new DbCompileError('JD0010',
1913
+ `strict mode refused a residual: '${forcing.construct}' — ${forcing.reason}`);
1914
+ }
1915
+ const retEntity = entry.planned.retEntity;
1916
+ if (register !== undefined && retEntity === null) {
1917
+ throw new DbCompileError('JD0034',
1918
+ 'a tracked cursor registers the entity documents it yields, and this document '
1919
+ + 'yields none: it returns a projection, a count or a window rather than one bare '
1920
+ + 'entity binding — read it untracked, or return the binding itself');
1921
+ }
1922
+ const each = register === undefined ? (item) => item : (item) => register(retEntity, item);
1923
+ const classified = cursorClass(entry, externals);
1924
+ refuseBuffered(options, classified, entities.get(entry.planned.retEntity ?? '')?.docPath);
1925
+ const signal = options?.signal;
1926
+ const deadline = options?.deadline;
1927
+ if (entry.planned.wrapped === true) {
1928
+ return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1929
+ materialize: () => chain(execute(document, options), (value) => [value]) });
1930
+ }
1931
+ if (classified.barrier !== null) {
1932
+ return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1933
+ materialize: () => chain(fetchRoot(entry), (root) =>
1934
+ packedResidualOf(entry, document)(root, externals).map(each)) });
1935
+ }
1936
+ const params = entry.slots.map((slot) => slotValue(slot, externals));
1937
+ // the statement is prepared by the first PULL, not here: a root
1938
+ // cursor's construction touches no connection, so it can be handed
1939
+ // back before the pull is admitted (MODEL-FORMAT §5.1)
1940
+ const prepared = () => {
1941
+ if (entry.statement === null) entry.statement = connection.prepare(entry.sql, { readOnly: true });
1942
+ return entry.statement;
1943
+ };
1944
+ if (entry.planned.plan.aggregate === 'count') {
1945
+ return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1946
+ materialize: () => chain(guardEntityScan(entry), () => chain(prepared(), (statement) =>
1947
+ chain(statement.get(params), (row) => [row?.value ?? 0]))) });
1948
+ }
1949
+ const rowEntity = entry.planned.plan.ret === null ? null
1950
+ : entry.planned.plan.bindings
1951
+ .find((binding) => binding.name === entry.planned.plan.ret).entity;
1952
+ let pulledRows = 0;
1953
+ // a statement of its own per cursor: two live iterators over one
1954
+ // cached statement invalidate each other at the driver
1955
+ return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
1956
+ open: () => chain(guardEntityScan(entry), () => chain(connection.prepare(entry.sql, { readOnly: true, ephemeral: true }),
1957
+ (statement) => statement.iterate(params))),
1958
+ items: (row) => {
1959
+ pulledRows++;
1960
+ if (entry.rowBound !== null && pulledRows > entry.rowBound) {
1961
+ throw new DbRuntimeError('JD2007',
1962
+ `the fetch crossed the profile's maxRows bound of ${entry.rowBound}`,
1963
+ { docPath: entities.get(rowEntity)?.docPath, collection: rowEntity });
1964
+ }
1965
+ const project = entry.planned.plan.project;
1966
+ if (project != null) return projectedTreeItems(project, row, 'p');
1967
+ return [each(checkBytes(entry, mergeEntityRow(mapping.entities[rowEntity], row, '__doc'), rowEntity))];
1968
+ } });
1079
1969
  };
1080
1970
 
1081
1971
  const explain = (document, options = undefined) => {
1082
- const pushdown = options?.pushdown !== false;
1083
- const entry = entryFor(document, pushdown);
1972
+ const { pushdown, profile, profileSource } = callState(options);
1973
+ const entry = entryFor(document, pushdown, profile);
1974
+ // bound against the externals it was given: a value the database
1975
+ // cannot take sends the run to the residual over the fetched root,
1976
+ // and that is the mode reported, with the diversion named first; no
1977
+ // externals at all is the abstract plan, as it always was
1978
+ const classified = cursorClass(entry, options?.externals ?? null);
1979
+ const diverted = entry.planned.mode === 'native' && classified.barrier !== null
1980
+ && classified.barrier.construct === 'external';
1981
+ const mode = diverted ? 'set' : entry.planned.mode;
1982
+ const reasons = diverted && classified.barrier !== null
1983
+ ? [classified.barrier, ...entry.planned.reasons] : entry.planned.reasons;
1084
1984
  const base = {
1085
- mode: entry.planned.mode,
1985
+ mode,
1986
+ streaming: classified.streaming,
1987
+ barrier: classified.barrier,
1988
+ budget: budgetOf(profile, profileSource, connection.capabilities),
1086
1989
  wrapped: entry.planned.wrapped === true,
1087
1990
  referenced: [...entry.planned.referenced],
1088
- reasons: entry.planned.reasons,
1089
- sql: entry.sql,
1090
- residual: entry.planned.mode === 'native'
1991
+ reasons,
1992
+ // the same effective-order vocabulary the collection engine and
1993
+ // the graph loader report; `null` when no statement answers
1994
+ order: diverted ? null : planOrder(entry.planned.plan),
1995
+ sql: diverted ? null : entry.sql,
1996
+ residual: mode === 'native'
1091
1997
  ? null
1092
- : { mode: 'set', reasons: entry.planned.reasons },
1998
+ : { mode: 'set', reasons },
1093
1999
  };
1094
- if (entry.planned.mode !== 'native') return base;
1095
- return chain(connection.prepare(dialect.explainQuery(entry.sql)), (statement) =>
2000
+ if (mode !== 'native') return base;
2001
+ return chain(connection.prepare(dialect.explainQuery(entry.sql), { readOnly: true }), (statement) =>
1096
2002
  chain(statement.all(entry.slots.map((slot) =>
1097
2003
  ('literal' in slot ? slot.literal : null))), (rows) => ({
1098
2004
  ...base,
1099
- join: entry.planned.plan.joinOn,
1100
- scanNarrative: rows.map((row) => String(row.detail)).join('; '),
2005
+ // the join graph, in the order the FROM clause builds it: the
2006
+ // first binding, then each one and the equalities that attached
2007
+ // it. Empty for a single-binding plan
2008
+ joins: entry.planned.plan.joins,
2009
+ scanNarrative: dialect.explainLines(rows).join('; '),
1101
2010
  })));
1102
2011
  };
1103
2012
 
1104
- return { execute, explain, relations };
2013
+ return { execute: bounded(execute, driverWrap), query,
2014
+ syncQuery: (document, options, register) => query(document, options, register, createSyncCursor),
2015
+ explain: bounded(explain, driverWrap), relations };
1105
2016
  }
1106
2017
 
1107
2018
  /**
@@ -1121,6 +2032,17 @@ export function createEntityQueryEngine(context) {
1121
2032
  */
1122
2033
  export function createLoadEngine(context, entityName) {
1123
2034
  const { connection, entities, mapping, state } = context;
2035
+ const storeProfile = context.profile ?? null;
2036
+ const roots = context.roots ?? [...entities.keys()];
2037
+ /** Every driver failure this loader meets, classified under its entity. */
2038
+ const driverWrap = (/** @type {any} */ error) =>
2039
+ wrapDriverError(error, { docPath: '/entities', collection: entityName });
2040
+ // the entity core's column encoding (booleans to integers, an epoch
2041
+ // column's string to its epoch): what a continuation's DOCUMENT values
2042
+ // bind as when the keyset compares them with the stored columns
2043
+ const encodeColumn = context.coreFor === undefined
2044
+ ? (/** @type {string} */ column, /** @type {any} */ value) => value
2045
+ : (column, value) => context.coreFor(entityName).plan.encodeColumn(column, value);
1124
2046
  // a registered operator (Ring 2) is recognised as vocabulary so a
1125
2047
  // where/orderBy that uses one refuses cleanly (JD0032 — the load path
1126
2048
  // is all-SQL, with no residual), never as an unknown operator
@@ -1132,40 +2054,48 @@ export function createLoadEngine(context, entityName) {
1132
2054
  `${reason} (include path: ${path.join('.') || '<root>'})`,
1133
2055
  entities.get(entityName)?.docPath);
1134
2056
  const isWindowBound = (value) => Number.isSafeInteger(value) && value >= 0;
2057
+ /** A per-root bound as declared: a positive integer, or `Infinity` /
2058
+ * `null` for the unbounded case a caller spelled on purpose. */
2059
+ const isBound = (value) => value === null || value === Infinity
2060
+ || (Number.isSafeInteger(value) && value >= 1);
1135
2061
  /** An include's window inside its subquery: LIMIT, and OFFSET for a
1136
- * `skip` — per parent row, since the subquery is correlated (§10.4). */
1137
- const windowClause = (child) => (child.take !== undefined || (child.skip !== undefined && child.skip > 0)
1138
- ? ` ${dialect.limitClause(child.take ?? null, child.skip)}` : '');
2062
+ * `skip` — per parent row, since the subquery is correlated (§10.4).
2063
+ * A to-many include with no `take` still carries `LIMIT maxRows + 1`,
2064
+ * so a relation past its bound is DETECTED at the bound instead of
2065
+ * aggregated whole and then refused — the profile's row-bound rule,
2066
+ * applied per root. */
2067
+ const windowClause = (child) => {
2068
+ const limit = child.take ?? (child.rowLimit === null ? null : child.rowLimit + 1);
2069
+ return limit !== null || (child.skip !== undefined && child.skip > 0)
2070
+ ? ` ${dialect.limitClause(limit, child.skip)}` : '';
2071
+ };
1139
2072
 
1140
2073
  /** Compile a where EXPRESSION over `$it` against one entity. */
1141
2074
  const compileWhere = (expression, entity, path) => {
1142
- const wrapper = { $for: { it: '$[*]' }, $where: expression, $return: '$it' };
1143
- let analysis;
1144
- try {
1145
- analysis = analyzeQuery(wrapper, analyzeOpts);
1146
- }
1147
- catch (cause) {
2075
+ const planned = planEntityWhere(expression, entity, mapping.entities[entity.name], analyzeOpts);
2076
+ if ('error' in planned) {
1148
2077
  throw new DbCompileError('JD0032',
1149
2078
  `the where expression does not compile (include path: ${path.join('.')})`,
1150
- entity.docPath, /** @type {Error} */ (cause));
2079
+ entity.docPath, planned.error);
1151
2080
  }
1152
- const flwor = analysis.root;
1153
- const slot = flwor.forBindings[0].slot;
1154
- const shape = entityShape(entity, mapping.entities[entity.name]);
1155
- const conjuncts = flwor.where.kind === 'op' && flwor.where.name === '$and'
1156
- ? flwor.where.args : [flwor.where];
1157
- let filter = null;
1158
- for (const conjunct of conjuncts) {
1159
- const outcome = planEntityPredicate(conjunct, slot, shape);
1160
- if ('refusal' in outcome) {
1161
- throw refuse(`the where expression is not translatable: ${outcome.refusal.reason}`, path);
1162
- }
1163
- filter = filter === null ? outcome.pred
1164
- : filter.p === 'and'
1165
- ? { p: 'and', items: [...filter.items, outcome.pred] }
1166
- : { p: 'and', items: [filter, outcome.pred] };
2081
+ if ('refusal' in planned) {
2082
+ throw refuse(`the where expression is not translatable: ${planned.refusal.reason}`, path);
2083
+ }
2084
+ return planned.filter;
2085
+ };
2086
+ /** The profile applied to one node of the include tree (§8): the
2087
+ * entity must be allowed, and its mandatory predicate — when the
2088
+ * profile carries one — is conjoined into the node's own filter, at
2089
+ * the root and inside every include subquery alike. */
2090
+ const applyProfileToNode = (node, name, profile, path) => {
2091
+ if (profile === null) return node;
2092
+ if (profile.collections !== null && !profile.collections.includes(name)) {
2093
+ throw profileEntityRefusal(`the profile does not allow loading entity '${name}'`
2094
+ + ` (include path: ${path.join('.') || '<root>'})`, entities.get(name)?.docPath);
1167
2095
  }
1168
- return filter;
2096
+ const predicate = mandatoryEntityPredicate(profile, entities.get(name), mapping.entities[name], analyzeOpts);
2097
+ if (predicate === null) return node;
2098
+ return { ...node, where: conjoin(node.where, predicate) };
1169
2099
  };
1170
2100
 
1171
2101
  const compileOrder = (orderBy, entity, path) => {
@@ -1194,12 +2124,13 @@ export function createLoadEngine(context, entityName) {
1194
2124
  };
1195
2125
 
1196
2126
  /** Build the include tree, validating names, depth and cycles. */
1197
- const buildTree = (name, spec, depth, maxDepth, path, seen) => {
2127
+ const buildTree = (name, spec, depth, maxDepth, path, seen, profile = null) => {
1198
2128
  const entity = entities.get(name);
1199
2129
  if (depth > maxDepth) {
1200
2130
  throw refuse(`the include graph exceeds its depth bound of ${maxDepth} `
1201
2131
  + '(raise it explicitly with maxDepth)', path);
1202
2132
  }
2133
+ /** @type {any} */
1203
2134
  const node = {
1204
2135
  entity,
1205
2136
  entityMapping: mapping.entities[name],
@@ -1207,10 +2138,14 @@ export function createLoadEngine(context, entityName) {
1207
2138
  order: spec?.orderBy !== undefined ? compileOrder(spec.orderBy, entity, path) : null,
1208
2139
  take: spec?.take,
1209
2140
  skip: spec?.skip,
2141
+ /** the per-root row bound a to-many include's subquery detects at;
2142
+ * set by the parent, `null` at the root and for a to-one */
2143
+ rowLimit: null,
1210
2144
  includes: [],
1211
2145
  };
2146
+ const shaped = applyProfileToNode(node, name, profile, path);
1212
2147
  const includeSpec = spec?.include;
1213
- if (includeSpec === undefined) return node;
2148
+ if (includeSpec === undefined) return shaped;
1214
2149
  if (seen.has(includeSpec))
1215
2150
  throw refuse('the include specification cycles', path);
1216
2151
  seen.add(includeSpec);
@@ -1225,7 +2160,7 @@ export function createLoadEngine(context, entityName) {
1225
2160
  if (childSpec.count === true) {
1226
2161
  // a count counts EVERY related row; a where/take beside it was
1227
2162
  // dropped without a word, and the number answered was the total
1228
- const dropped = ['where', 'orderBy', 'take', 'skip', 'include', 'after']
2163
+ const dropped = ['where', 'orderBy', 'take', 'skip', 'include', 'after', 'maxRows', 'maxBytes']
1229
2164
  .filter((member) => childSpec[member] !== undefined);
1230
2165
  if (dropped.length > 0) {
1231
2166
  throw refuse(`count: true counts every related row and takes no ${dropped.join('/')} — `
@@ -1236,6 +2171,11 @@ export function createLoadEngine(context, entityName) {
1236
2171
  if (childSpec[member] !== undefined && !isWindowBound(childSpec[member]))
1237
2172
  throw refuse(`${member} must be a non-negative integer`, [...path, relationName]);
1238
2173
  }
2174
+ for (const member of ['maxRows', 'maxBytes']) {
2175
+ if (childSpec[member] !== undefined && !isBound(childSpec[member]))
2176
+ throw refuse(`${member} must be a positive integer, or Infinity (null in JSON) to load the relation unbounded by decision`,
2177
+ [...path, relationName]);
2178
+ }
1239
2179
  // a keyset cursor is one position in ONE ordered set; an include is
1240
2180
  // a set per parent, so it windows with skip/take and never seeks
1241
2181
  if (childSpec.after !== undefined) {
@@ -1243,11 +2183,32 @@ export function createLoadEngine(context, entityName) {
1243
2183
  [...path, relationName]);
1244
2184
  }
1245
2185
  const childName = relation.to;
2186
+ const many = relation.kind !== 'oneToOne';
2187
+ // the per-root bounds (§10.4): declared, else the include's own
2188
+ // `take` (a window IS a row bound), else the store default; `null`
2189
+ // is the unbounded case, spelled
2190
+ const boundOf = (member, fallback) => {
2191
+ const declared = childSpec[member];
2192
+ if (declared === undefined) return fallback;
2193
+ return declared === Infinity ? null : declared;
2194
+ };
2195
+ const maxRows = many ? boundOf('maxRows', childSpec.take ?? INCLUDE_ROWS_DEFAULT) : null;
2196
+ if (profile !== null && profile.maxIncludedRows !== null && many && childSpec.count !== true
2197
+ && (maxRows === null || maxRows > profile.maxIncludedRows)) {
2198
+ // the profile's cap on included rows per root is a hard maximum
2199
+ // the include's own declaration cannot exceed (D6: refused, not
2200
+ // narrowed quietly)
2201
+ throw profileEntityRefusal(`the profile caps included rows per root at ${profile.maxIncludedRows}; `
2202
+ + `the include '${relationName}' declares ${maxRows === null ? 'no bound (Infinity)' : maxRows}`
2203
+ + ` (include path: ${[...path, relationName].join('.')})`, entity.docPath);
2204
+ }
1246
2205
  const include = {
1247
2206
  name: relationName,
1248
2207
  field: `__${relationName}`,
1249
2208
  relation,
1250
- many: relation.kind !== 'oneToOne',
2209
+ many,
2210
+ maxRows,
2211
+ maxBytes: childSpec.count === true ? null : boundOf('maxBytes', INCLUDE_BYTES_DEFAULT),
1251
2212
  count: childSpec.count === true,
1252
2213
  // the join kind is derivable from the schema: a required
1253
2214
  // foreign key means the parent always exists
@@ -1258,11 +2219,13 @@ export function createLoadEngine(context, entityName) {
1258
2219
  child: childSpec.count === true
1259
2220
  ? null
1260
2221
  : buildTree(childName, childSpec, depth + 1, maxDepth,
1261
- [...path, relationName], seen),
2222
+ [...path, relationName], seen, profile),
1262
2223
  };
1263
- node.includes.push(include);
2224
+ // the subquery's own LIMIT detects the bound (windowClause)
2225
+ if (include.child !== null) include.child.rowLimit = many ? include.maxRows : null;
2226
+ shaped.includes.push(include);
1264
2227
  }
1265
- return node;
2228
+ return shaped;
1266
2229
  };
1267
2230
 
1268
2231
  /** Render one node's subquery-projection SQL. */
@@ -1280,7 +2243,10 @@ export function createLoadEngine(context, entityName) {
1280
2243
  if (named.has(fk.column)) continue; // a declared via property
1281
2244
  parts.push(`${slText(fk.column)}, ${aliasSql}.${q(fk.column)}`);
1282
2245
  }
1283
- parts.push(`${slText('__doc')}, ${dialect.jsonText(docSql)}`);
2246
+ // EMBEDDED, not rendered: the enclosing object carries the
2247
+ // document as a nested JSON value, which on an engine with one
2248
+ // JSON type is the column itself and on SQLite is `json()`
2249
+ parts.push(`${slText('__doc')}, ${dialect.jsonEmbed(docSql)}`);
1284
2250
  for (const include of node.includes)
1285
2251
  parts.push(`${slText(include.field)}, ${renderInclude(node, include, alias, param, emitters)}`);
1286
2252
  return parts.join(', ');
@@ -1327,7 +2293,7 @@ export function createLoadEngine(context, entityName) {
1327
2293
  // engine's order); only plain mapped columns order natively
1328
2294
  const value = term.ref.flavor === 'entity-column'
1329
2295
  ? `${rendered.aliasSql}.${q(term.ref.column)}`
1330
- : dialect.jsonExtract(rendered.docSql, dialect.jsonPathText(term.ref.segments));
2296
+ : dialect.jsonExtract(rendered.docSql, dialect.jsonPathText(term.ref.segments), 'text');
1331
2297
  const nullsFirst = term.emptyGreatest === term.desc;
1332
2298
  return `${value} ${term.desc ? 'DESC' : 'ASC'}${dialect.orderNulls(nullsFirst)}`;
1333
2299
  });
@@ -1345,17 +2311,98 @@ export function createLoadEngine(context, entityName) {
1345
2311
  + `WHERE ${conditions.join(' AND ')} ORDER BY ${orderSql.join(', ')}`
1346
2312
  + windowClause(child);
1347
2313
  if (relation.kind === 'oneToOne') {
1348
- return `(SELECT json_object(${rendered.projection()}) FROM `
2314
+ return `(SELECT ${dialect.jsonObject(rendered.projection())} FROM `
1349
2315
  + `(${inner} ${dialect.limitClause(1, undefined)}) AS ${q(childAlias)})`;
1350
2316
  }
1351
- return `(SELECT ${dialect.jsonAgg(`json_object(${rendered.projection()})`)} `
2317
+ return `(SELECT ${dialect.jsonAgg(dialect.jsonObject(rendered.projection()))} `
1352
2318
  + `FROM (${inner}) AS ${q(childAlias)})`;
1353
2319
  };
1354
2320
 
1355
- const buildLoad = (spec) => {
2321
+ /**
2322
+ * The ordering's IDENTITY in keyset mode: the declared column terms
2323
+ * with their direction and null placement, then the primary-key
2324
+ * column(s) not already named, ascending — the tie-breaker the plan
2325
+ * appends whether or not the caller named it, since the key is the
2326
+ * one column guaranteed unique. A document-path term cannot carry a
2327
+ * keyset (`JD0032`).
2328
+ * @param {any[]} order - the compiled order terms
2329
+ * @param {readonly string[]} keyColumns
2330
+ * @returns {{ column: string, desc: boolean, nullsFirst: boolean }[]}
2331
+ */
2332
+ const orderIdentity = (order, keyColumns) => {
2333
+ const terms = order.map((term) => {
2334
+ if (term.ref.flavor !== 'entity-column') {
2335
+ throw refuse('a keyset orders by mapped columns — '
2336
+ + `'${term.ref.segments.join('.')}' is a document path`, []);
2337
+ }
2338
+ return { column: term.ref.column, desc: term.desc, nullsFirst: term.emptyGreatest === term.desc };
2339
+ });
2340
+ for (const column of keyColumns) {
2341
+ // a key column is NOT NULL: `nullsFirst` is SQLite's own ASC
2342
+ // default, spelled so the identity is explicit, never a branch
2343
+ if (!terms.some((term) => term.column === column))
2344
+ terms.push({ column, desc: false, nullsFirst: true });
2345
+ }
2346
+ return terms;
2347
+ };
2348
+
2349
+ /**
2350
+ * A structural continuation, checked against THIS ordering: the
2351
+ * `{ order, keys, key }` a page emitted, whose `order` must be this
2352
+ * graph's identity exactly — a continuation replayed against another
2353
+ * ordering is `JD0035`, never a wrong page — and whose values are
2354
+ * returned aligned with the identity, encoded as the columns store them.
2355
+ * @param {any} after
2356
+ * @param {ReturnType<typeof orderIdentity>} identity
2357
+ * @param {number} declared - how many terms the caller declared
2358
+ * @param {readonly string[]} keyColumns
2359
+ * @returns {any[]} one value per identity term
2360
+ */
2361
+ const continuationValues = (after, identity, declared, keyColumns) => {
2362
+ const mismatch = (reason) => new DbCompileError('JD0035',
2363
+ `the continuation does not belong to this ordering: ${reason}`,
2364
+ entities.get(entityName)?.docPath);
2365
+ const spell = (terms) => terms.map((term) => `${term.column} ${term.desc ? 'desc' : 'asc'}`
2366
+ + `${term.nullsFirst ? ' nulls first' : ''}`).join(', ');
2367
+ if (after === null || typeof after !== 'object' || !Array.isArray(after.order)
2368
+ || !Array.isArray(after.keys) || !('key' in after)) {
2369
+ throw mismatch('a continuation is the { order, keys, key } value a page emitted');
2370
+ }
2371
+ if (JSON.stringify(after.order) !== JSON.stringify(identity)) {
2372
+ throw mismatch(`it was emitted for the ordering (${spell(after.order)}); this graph orders `
2373
+ + `by (${spell(identity)})`);
2374
+ }
2375
+ if (after.keys.length !== declared) {
2376
+ throw mismatch(`it carries ${after.keys.length} order-key value(s); the ordering declares ${declared}`);
2377
+ }
2378
+ const keyOf = (column) => {
2379
+ if (keyColumns.length === 1) {
2380
+ if (typeof after.key !== 'string' && typeof after.key !== 'number')
2381
+ throw mismatch("'key' must be the row's primary key, a scalar");
2382
+ return after.key;
2383
+ }
2384
+ const value = after.key?.[column];
2385
+ if (typeof value !== 'string' && typeof value !== 'number')
2386
+ throw mismatch(`'key' must carry every key column { ${keyColumns.join(', ')} }`);
2387
+ return value;
2388
+ };
2389
+ return identity.map((term, i) => encodeColumn(term.column,
2390
+ i < declared ? (after.keys[i] ?? null) : keyOf(term.column)));
2391
+ };
2392
+
2393
+ /**
2394
+ * Build the load: one statement, cached by spec. `keyset` forces
2395
+ * keyset mode — the primary key as the ORDER BY tie-breaker in place of
2396
+ * the row identity, so a continuation can resume exactly — which a
2397
+ * structural `after` implies; a scalar `after` keeps the single
2398
+ * unique-column keyset it always was.
2399
+ * @param {any} spec
2400
+ * @param {boolean} [keyset]
2401
+ */
2402
+ const buildLoad = (spec, keyset = false, profile = null) => {
1356
2403
  // a cyclic specification cannot be keyed, so the cache reports a
1357
2404
  // permanent miss and buildTree gets to NAME the cycle
1358
- const key = ['L', entityName, spec ?? {}, dialect.name];
2405
+ const key = ['L', entityName, spec ?? {}, dialect.name, keyset, profile];
1359
2406
  const cached = state.cache.get(key);
1360
2407
  if (cached !== undefined) {
1361
2408
  state.counters.hits++;
@@ -1370,12 +2417,16 @@ export function createLoadEngine(context, entityName) {
1370
2417
  };
1371
2418
  const emitters = createEntityPredicateEmitters(dialect, param);
1372
2419
  const maxDepth = spec?.maxDepth ?? INCLUDE_DEPTH_DEFAULT;
2420
+ if (profile !== null && profile.maxDepth !== null && maxDepth > profile.maxDepth) {
2421
+ throw profileEntityRefusal(`the profile caps the include depth at ${profile.maxDepth}; `
2422
+ + `this load asks for ${maxDepth}`, entities.get(entityName)?.docPath);
2423
+ }
1373
2424
  for (const member of ['take', 'skip']) {
1374
2425
  // interpolated into LIMIT/OFFSET as written: a string ran as SQL
1375
2426
  if (spec?.[member] !== undefined && !isWindowBound(spec[member]))
1376
2427
  throw refuse(`${member} must be a non-negative integer`, []);
1377
2428
  }
1378
- const tree = buildTree(entityName, spec ?? {}, 0, maxDepth, [], new Set());
2429
+ const tree = buildTree(entityName, spec ?? {}, 0, maxDepth, [], new Set(), profile);
1379
2430
  const rendered = render(tree, 'r', param, emitters);
1380
2431
 
1381
2432
  // anonymous placeholders bind by position, so slots must be
@@ -1388,16 +2439,69 @@ export function createLoadEngine(context, entityName) {
1388
2439
  if (tree.where !== null)
1389
2440
  conditions.push(emitters.emitPred(rendered.aliasSql, rendered.docSql, tree.where));
1390
2441
 
1391
- // pagination: keyset over a single unique ordering column beats a
1392
- // growing OFFSET; the choice is reported, never silent
2442
+ // pagination: keyset beats a growing OFFSET; the choice is reported,
2443
+ // never silent. A scalar `after` is the single unique-column keyset;
2444
+ // a structural one — or a page — is the composite keyset: the
2445
+ // lexicographic expansion over the declared terms with the primary
2446
+ // key appended, null placement agreeing with the ORDER BY (§10.5)
1393
2447
  let pagination = 'none';
1394
2448
  const order = tree.order ?? [];
2449
+ const after = spec?.after === null ? undefined : spec?.after;
2450
+ const structural = after !== undefined && typeof after === 'object';
2451
+ const keysetMode = keyset || structural;
2452
+ const keyColumns = tree.entityMapping.keys;
1395
2453
  const uniqueColumns = new Set([
1396
- tree.entityMapping.keys.length === 1 ? tree.entityMapping.keys[0] : null,
2454
+ keyColumns.length === 1 ? keyColumns[0] : null,
1397
2455
  ...tree.entityMapping.indexes.filter((index) => index.unique)
1398
2456
  .map((index) => index.property),
1399
2457
  ]);
1400
- if (spec?.after !== undefined) {
2458
+ /** @type {ReturnType<typeof orderIdentity> | null} */
2459
+ let identity = null;
2460
+ if (keysetMode) {
2461
+ identity = orderIdentity(order, keyColumns);
2462
+ if (after !== undefined) {
2463
+ if (!structural) {
2464
+ throw new DbCompileError('JD0035',
2465
+ 'the continuation does not belong to this ordering: a page resumes from the '
2466
+ + '{ order, keys, key } value a page emitted, not a bare key',
2467
+ entities.get(entityName)?.docPath);
2468
+ }
2469
+ pagination = 'keyset';
2470
+ const values = continuationValues(after, identity, order.length, keyColumns);
2471
+ const column = (term) => `${rendered.aliasSql}.${q(term.column)}`;
2472
+ const equal = (i) => (values[i] === null
2473
+ ? `${column(identity[i])} IS NULL`
2474
+ : `${column(identity[i])} = ${param({ literal: values[i] })}`);
2475
+ // "comes after the value in this term's order": a null value is
2476
+ // followed by the non-nulls when nulls sort first and by nothing
2477
+ // when they sort last; a non-null value is followed by the greater
2478
+ // (or lesser, descending) values, and by the nulls when they sort
2479
+ // last — the comparison alone would drop them, since SQL's
2480
+ // `col > ?` is neither true nor false for NULL
2481
+ const beyond = (i) => {
2482
+ const term = identity[i];
2483
+ if (values[i] === null) return `${column(term)} IS NOT NULL`;
2484
+ const base = `${column(term)} ${term.desc ? '<' : '>'} ${param({ literal: values[i] })}`;
2485
+ // a key column is NOT NULL and needs no null branch
2486
+ return term.nullsFirst || keyColumns.includes(term.column)
2487
+ ? base : `(${base} OR ${column(term)} IS NULL)`;
2488
+ };
2489
+ const branches = [];
2490
+ for (let i = 0; i < identity.length; i++) {
2491
+ // after a null that sorts last comes nothing in this term: the
2492
+ // branch is empty, and only the tie-break branches remain
2493
+ if (values[i] === null && !identity[i].nullsFirst) continue;
2494
+ // parameters bind by position, so the parts are built in SQL
2495
+ // text order: the equalities first, then the strict comparison
2496
+ const parts = [];
2497
+ for (let j = 0; j < i; j++) parts.push(equal(j));
2498
+ parts.push(beyond(i));
2499
+ branches.push(parts.length === 1 ? parts[0] : `(${parts.join(' AND ')})`);
2500
+ }
2501
+ conditions.push(branches.length === 0 ? '0' : `(${branches.join(' OR ')})`);
2502
+ }
2503
+ }
2504
+ else if (after !== undefined) {
1401
2505
  const term = order.length === 1 ? order[0] : null;
1402
2506
  if (term === null || term.ref.flavor === 'entity-doc'
1403
2507
  || !uniqueColumns.has(term.ref.column)) {
@@ -1405,7 +2509,7 @@ export function createLoadEngine(context, entityName) {
1405
2509
  }
1406
2510
  pagination = 'keyset';
1407
2511
  conditions.push(`${rendered.aliasSql}.${q(term.ref.column)} `
1408
- + `${term.desc ? '<' : '>'} ${param({ literal: spec.after })}`);
2512
+ + `${term.desc ? '<' : '>'} ${param({ literal: after })}`);
1409
2513
  }
1410
2514
  else if (spec?.skip !== undefined && spec.skip > 0) {
1411
2515
  pagination = 'offset';
@@ -1415,53 +2519,260 @@ export function createLoadEngine(context, entityName) {
1415
2519
  + includeSql
1416
2520
  + ` FROM ${q(tree.entityMapping.table)} AS ${rendered.aliasSql}`;
1417
2521
  if (conditions.length > 0) sql += ` WHERE ${conditions.join(' AND ')}`;
1418
- const orderSql = order.map((term) => {
1419
- const value = term.ref.flavor === 'entity-column'
1420
- ? `${rendered.aliasSql}.${q(term.ref.column)}`
1421
- : dialect.jsonExtract(rendered.docSql, dialect.jsonPathText(term.ref.segments));
1422
- const nullsFirst = term.emptyGreatest === term.desc;
1423
- return `${value} ${term.desc ? 'DESC' : 'ASC'}${dialect.orderNulls(nullsFirst)}`;
1424
- });
1425
- orderSql.push(`${rendered.aliasSql}.${dialect.rowIdentity()}`);
2522
+ const orderSql = identity !== null
2523
+ ? identity.map((term) => `${rendered.aliasSql}.${q(term.column)} `
2524
+ + `${term.desc ? 'DESC' : 'ASC'}${dialect.orderNulls(term.nullsFirst)}`)
2525
+ : order.map((term) => {
2526
+ const value = term.ref.flavor === 'entity-column'
2527
+ ? `${rendered.aliasSql}.${q(term.ref.column)}`
2528
+ : dialect.jsonExtract(rendered.docSql, dialect.jsonPathText(term.ref.segments));
2529
+ const nullsFirst = term.emptyGreatest === term.desc;
2530
+ return `${value} ${term.desc ? 'DESC' : 'ASC'}${dialect.orderNulls(nullsFirst)}`;
2531
+ });
2532
+ if (identity === null) orderSql.push(`${rendered.aliasSql}.${dialect.rowIdentity()}`);
1426
2533
  sql += ` ORDER BY ${orderSql.join(', ')}`;
1427
- if (spec?.take !== undefined || pagination === 'offset') {
1428
- sql += ` ${dialect.limitClause(spec?.take ?? null,
2534
+ // the profile's row bound rides the root as LIMIT maxRows + 1, so
2535
+ // a load past it is detected at the bound and refused (JD2007)
2536
+ const rowBound = profile === null ? null : profile.maxRows;
2537
+ const take = rowBound === null ? spec?.take
2538
+ : Math.min(spec?.take ?? Infinity, rowBound + 1);
2539
+ if (take !== undefined || pagination === 'offset') {
2540
+ sql += ` ${dialect.limitClause(take === undefined ? null : take,
1429
2541
  pagination === 'offset' ? spec.skip : undefined)}`;
1430
2542
  }
1431
2543
 
1432
- const entry = { sql, slots, tree, pagination, statement: null };
2544
+ const entry = {
2545
+ sql, slots, tree, pagination, statement: null,
2546
+ rowBound,
2547
+ byteBound: profile === null ? null : profile.maxBytes,
2548
+ profile,
2549
+ identity: identity === null ? null : deepFreeze(identity),
2550
+ // the order the statement executes under, in the vocabulary both
2551
+ // query engines report: the declared terms and the tie-breaker the
2552
+ // ORDER BY above appends — the primary key in keyset mode, the row
2553
+ // identity otherwise. Built from the same normalized terms, so the
2554
+ // explanation cannot drift from the clause
2555
+ effective: deepFreeze(effectiveOrder(order,
2556
+ identity === null ? undefined : { keyColumns })),
2557
+ declared: order.map((term) => term.ref.column),
2558
+ keyColumns,
2559
+ // a page over this ordering is a snapshot only when every order key
2560
+ // is immutable, and the primary key is the one column the engine
2561
+ // itself guarantees never moves (`update()` refuses to rewrite it)
2562
+ snapshot: identity === null ? null : identity.every((term) => keyColumns.includes(term.column)),
2563
+ };
1433
2564
  const sizeBefore = state.cache.size();
1434
2565
  if (state.cache.set(key, entry) && state.cache.size() === sizeBefore)
1435
2566
  state.counters.evictions++;
1436
2567
  return entry;
1437
2568
  };
1438
2569
 
1439
- return {
2570
+ /**
2571
+ * A graph load answers whole entity documents, and a member
2572
+ * allow-list cannot cover a whole document — so a policed root is
2573
+ * refused here rather than answered past its policy. The refusal
2574
+ * names the members that ARE allowed, because the document query
2575
+ * engine can project exactly those.
2576
+ * @param {any} profile
2577
+ */
2578
+ const refuseMemberPolicy = (profile) => {
2579
+ const policy = profile === null || profile.members === null
2580
+ ? undefined : profile.members[entityName];
2581
+ if (policy === undefined) return;
2582
+ throw new DbCompileError('JD0011',
2583
+ `the profile allows only the members (${policy.declared.join(', ')}) of `
2584
+ + `'${entityName}', and a graph load answers whole documents — query the members `
2585
+ + 'the policy allows instead', entities.get(entityName)?.docPath);
2586
+ };
2587
+
2588
+ /** One loaded root against the profile's row and byte bounds. */
2589
+ const checkRoot = (entry, doc, pulled) => {
2590
+ if (entry.rowBound !== null && pulled > entry.rowBound) {
2591
+ throw new DbRuntimeError('JD2007',
2592
+ `the load crossed the profile's maxRows bound of ${entry.rowBound}`,
2593
+ { docPath: entities.get(entityName)?.docPath, collection: entityName });
2594
+ }
2595
+ if (entry.byteBound !== null) {
2596
+ const bytes = utf8Length(JSON.stringify(doc));
2597
+ if (bytes > entry.byteBound) {
2598
+ throw new DbRuntimeError('JD2076',
2599
+ `an item of ${bytes} serialised bytes exceeds the profile's maxBytes bound of ${entry.byteBound}`,
2600
+ { docPath: entities.get(entityName)?.docPath, collection: entityName });
2601
+ }
2602
+ }
2603
+ return doc;
2604
+ };
2605
+ /** The profile one call resolves, as on the query engines. */
2606
+ const profileOf = (options) => {
2607
+ const resolved = options?.profile !== undefined
2608
+ ? normalizeProfile(options.profile) : storeProfile;
2609
+ assertProfileRoots(resolved, roots, entities.get(entityName)?.docPath);
2610
+ refuseMemberPolicy(resolved);
2611
+ return resolved;
2612
+ };
2613
+ const profileSourceOf = (options) => (options?.profile !== undefined ? 'call'
2614
+ : (storeProfile === null ? null : 'store'));
2615
+
2616
+ /** The graph cursor over one built load: one root row per pull. */
2617
+ const openCursor = (entry, signal, register, deadline = undefined, cursorFactory = createCursor) => {
2618
+ const params = entry.slots.map((slot) => slot.literal);
2619
+ const each = register === undefined ? (doc) => doc : (doc) => register(entry.tree, doc);
2620
+ let pulled = 0;
2621
+ // prepared by the first pull, never at construction (MODEL-FORMAT
2622
+ // §5.1), and a statement of this cursor's own: two live iterators
2623
+ // over one cached statement invalidate each other at the driver
2624
+ return cursorFactory({ ...rowClassOf(connection), signal, deadline, now: state.now, wrap: driverWrap,
2625
+ open: () => chain(connection.prepare(entry.sql, { readOnly: true, ephemeral: true }), (statement) => statement.iterate(params)),
2626
+ items: (row) => [each(checkRoot(entry, parseGraphRow(entry.tree, row, '__doc'), ++pulled))] });
2627
+ };
2628
+
2629
+ /**
2630
+ * The continuation one root emits: unsigned, structural, opaque —
2631
+ * the ordering's identity (so it cannot be replayed against another
2632
+ * ordering), the declared order-key values as the DOCUMENT carries
2633
+ * them, and the row's primary key, the tie-breaker. Signing, tenant
2634
+ * scoping, expiry and wire encoding are the host's: the store has no
2635
+ * principal and no key, and a signature it invented would be theatre.
2636
+ */
2637
+ const continuationOf = (entry, doc) => deepFreeze({
2638
+ order: entry.identity,
2639
+ keys: entry.declared.map((column) => doc[column] ?? null),
2640
+ key: entry.keyColumns.length === 1
2641
+ ? doc[entry.keyColumns[0]]
2642
+ : Object.fromEntries(entry.keyColumns.map((column) => [column, doc[column]])),
2643
+ });
2644
+
2645
+ const surface = {
1440
2646
  treeFor(spec) {
1441
2647
  return buildLoad(spec).tree;
1442
2648
  },
1443
- load(spec) {
1444
- const entry = buildLoad(spec);
1445
- if (entry.statement === null) entry.statement = connection.prepare(entry.sql);
2649
+ load(spec, options = undefined) {
2650
+ requireCallable(options, state.now);
2651
+ const entry = buildLoad(spec, false, profileOf(options));
2652
+ if (entry.statement === null) entry.statement = connection.prepare(entry.sql, { readOnly: true });
1446
2653
  const params = entry.slots.map((slot) => slot.literal);
1447
2654
  return chain(entry.statement, (statement) =>
1448
2655
  chain(statement.all(params), (rows) =>
1449
- rows.map((row) => parseGraphRow(entry.tree, row, '__doc'))));
2656
+ rows.map((row, i) => checkRoot(entry, parseGraphRow(entry.tree, row, '__doc'), i + 1))));
1450
2657
  },
1451
- explainLoad(spec) {
1452
- const entry = buildLoad(spec);
2658
+ /**
2659
+ * The graph cursor: ONE root graph per pull, its includes attached
2660
+ * and bounded, from the same one statement `load` runs — the include
2661
+ * rows ride inside each root row as the JSON the database projected,
2662
+ * so the window is the row itself and no second statement per level
2663
+ * exists to hold or release. `register`, when given, is the unit of
2664
+ * work's graph registration, applied per root as it is yielded.
2665
+ * @param {any} spec
2666
+ * @param {{ signal?: AbortSignal }} [options]
2667
+ * @param {((tree: any, doc: any) => any) | undefined} [register]
2668
+ */
2669
+ loadCursor(spec, options = undefined, register = undefined, cursorFactory = createCursor) {
2670
+ requireCallable(options, state.now);
2671
+ return openCursor(buildLoad(spec, false, profileOf(options)), options?.signal, register,
2672
+ options?.deadline, cursorFactory);
2673
+ },
2674
+ /**
2675
+ * One page: a bounded drain of the graph cursor in keyset mode —
2676
+ * `limit` roots at most, `maxBytes` serialised bytes at most, the
2677
+ * continuation of the last delivered root, `hasMore` by one peek —
2678
+ * plus `snapshot`, true only over an immutable ordering (§10.5).
2679
+ * `consistency: 'snapshot'` over a mutable ordering is refused
2680
+ * (`JD0036`) rather than mislabelled; the default `'live'` reports
2681
+ * the truth either way.
2682
+ * @param {any} spec
2683
+ * @param {{ limit?: number, after?: any, maxBytes?: number | null,
2684
+ * consistency?: 'live' | 'snapshot', signal?: AbortSignal }} [options]
2685
+ * @param {((tree: any, doc: any) => any) | undefined} [register]
2686
+ */
2687
+ page(spec, options = undefined, register = undefined, cursorFactory = createCursor) {
2688
+ requireCallable(options, state.now);
2689
+ const limit = options?.limit ?? PAGE_LIMIT_DEFAULT;
2690
+ if (!Number.isSafeInteger(limit) || limit < 1)
2691
+ throw refuse('page() limit must be a positive integer', []);
2692
+ const declaredBytes = options?.maxBytes;
2693
+ const maxBytes = declaredBytes === undefined || declaredBytes === null || declaredBytes === Infinity
2694
+ ? null : declaredBytes;
2695
+ if (maxBytes !== null && !(Number.isSafeInteger(maxBytes) && maxBytes >= 1))
2696
+ throw refuse('page() maxBytes must be a positive integer, or Infinity for no byte bound', []);
2697
+ const consistency = options?.consistency ?? 'live';
2698
+ if (consistency !== 'live' && consistency !== 'snapshot')
2699
+ throw refuse("page() consistency is 'live' or 'snapshot'", []);
2700
+ if (spec?.take !== undefined || spec?.skip !== undefined)
2701
+ throw refuse('page() windows by its limit and continuation — a take or skip in the spec is refused', []);
2702
+ const after = options?.after ?? spec?.after ?? undefined;
2703
+ const paged = { ...(spec ?? {}), take: limit + 1 };
2704
+ if (after === undefined) delete paged.after;
2705
+ else paged.after = after;
2706
+ const entry = buildLoad(paged, true, profileOf(options));
2707
+ if (consistency === 'snapshot' && entry.snapshot !== true) {
2708
+ throw new DbCompileError('JD0036',
2709
+ `a snapshot page needs an ordering over immutable keys; this graph orders by (${
2710
+ entry.declared.join(', ')}), which a write may change, so it is LIVE pagination — a `
2711
+ + 'row whose order key changes can move across the cursor. Order by the primary key, '
2712
+ + "or ask for consistency: 'live' and read snapshot: false",
2713
+ entities.get(entityName)?.docPath);
2714
+ }
2715
+ // the drain peeks one root past the page to decide `hasMore`, so
2716
+ // registration happens on the DELIVERED roots after the drain — a
2717
+ // peeked root the caller never received must not enter the unit
2718
+ // of work
2719
+ const cursor = openCursor(entry, options?.signal, undefined, options?.deadline, cursorFactory);
2720
+ return chain(drainPage(cursor, {
2721
+ limit, maxBytes, after: after ?? null,
2722
+ sizeOf: (doc) => utf8Length(JSON.stringify(doc)),
2723
+ continuationOf: (doc) => continuationOf(entry, doc),
2724
+ }), (page) => ({
2725
+ ...page,
2726
+ items: register === undefined ? page.items : page.items.map((doc) => register(entry.tree, doc)),
2727
+ snapshot: entry.snapshot === true,
2728
+ }));
2729
+ },
2730
+ explainLoad(spec, options = undefined) {
2731
+ const entry = buildLoad(spec, spec?.after !== undefined && typeof spec.after === 'object', profileOf(options));
1453
2732
  const describe = (node, path) => node.includes.flatMap((include) => [
1454
2733
  { path: [...path, include.name].join('.'), kind: include.kind,
1455
2734
  count: include.count === true },
1456
2735
  ...(include.child === null ? [] : describe(include.child, [...path, include.name])),
1457
2736
  ]);
2737
+ // the per-root bounds every include runs under (§10.4): `null` is
2738
+ // the unbounded case a caller spelled; a count carries none
2739
+ const bounds = (node, path) => node.includes.flatMap((include) => [
2740
+ ...(include.count === true ? [] : [{ path: [...path, include.name].join('.'),
2741
+ maxRows: include.maxRows, maxBytes: include.maxBytes }]),
2742
+ ...(include.child === null ? [] : bounds(include.child, [...path, include.name])),
2743
+ ]);
1458
2744
  return {
1459
2745
  sql: entry.sql,
1460
2746
  pagination: entry.pagination,
1461
2747
  includes: describe(entry.tree, []),
2748
+ bounds: bounds(entry.tree, []),
2749
+ // the effective deterministic order the statement executes
2750
+ // under, in every load mode — a load outside keyset mode orders
2751
+ // by its declared terms and the row identity, and saying so is
2752
+ // not the same as having no order at all
2753
+ order: entry.effective,
2754
+ // the keyset ordering's IDENTITY, the value a continuation
2755
+ // carries and is checked against (`JD0035`) — `null` for a load
2756
+ // that is not in keyset mode, which has no continuation to emit
2757
+ identity: entry.identity,
2758
+ snapshot: entry.snapshot,
2759
+ // a graph load pulls one root row per statement row — where the
2760
+ // binding can hand rows over one at a time
2761
+ ...rowClassOf(connection),
2762
+ // the profile that applied and every bound it imposed (D7)
2763
+ budget: budgetOf(entry.profile, profileSourceOf(options), connection.capabilities),
1462
2764
  };
1463
2765
  },
1464
2766
  };
2767
+ // the loader's members answer through the boundary: a driver failure
2768
+ // arrives classified, a coded refusal as it is; the cursors carry the
2769
+ // same wrap
2770
+ return { ...surface,
2771
+ load: bounded(surface.load, driverWrap),
2772
+ syncLoadCursor: (spec, options, register) => surface.loadCursor(spec, options, register, createSyncCursor),
2773
+ syncPage: bounded((spec, options, register) => surface.page(spec, options, register, createSyncCursor), driverWrap),
2774
+ page: bounded(surface.page, driverWrap),
2775
+ explainLoad: bounded(surface.explainLoad, driverWrap) };
1465
2776
  }
1466
2777
 
1467
2778
  /**