@noy-db/hub 0.7.0 → 0.8.0-pre.0

package/dist/kernel/query/relate/explain.d.ts

@@ -0,0 +1,199 @@
+/**
+ * `Query.explain()` — a readable execution plan (#1348).
+ *
+ * `toPlan()` answers *what* the query is; this answers *how it would run*:
+ * the dispatch per clause (`index:hash` vs `scan`), row estimates taken from
+ * real index cardinalities, join strategy and side sizes against the row
+ * ceiling, whether ordering/pagination lands pre- or post-join, and which
+ * cardinality caps are close to tripping.
+ *
+ * ⛔ PURELY OBSERVATIONAL. Every read here is a size probe or an index
+ * bucket lookup — both pure. `explain()` must never execute the plan, and a
+ * terminal must return the same thing whether or not it was called. The
+ * guarantee is pinned by `__tests__/query-explain.test.ts`
+ * ("results are identical whether or not explain() was called").
+ *
+ * ⚠️ DISPATCH SELECTION IS MIRRORED, NOT SHARED. `indexDispatchFor` below
+ * reproduces `candidateRecords()` in `builder.ts` — deliberately, so this
+ * file stays out of the executor's way. That mirror is not left to trust:
+ * the index path never calls `source.snapshot()` and the scan path always
+ * does, so a snapshot-call counter is an executor-side witness of dispatch,
+ * and the suite asserts the two agree. **A new index kind must land in BOTH
+ * places** — a branch here and the matching branch in `candidateRecords` —
+ * plus a label on {@link ExplainDispatch}.
+ *
+ * ⭐ #1375 — THE MIRROR IS NOW POLICED EXHAUSTIVELY, and that is the durable
+ * half of the fix. `__tests__/query-explain.test.ts` carries a TABLE of query
+ * shapes spanning every dispatch kind and asserts, for each, that the label
+ * `explain()` claims agrees with an executor-side observation. Add a shape to
+ * that table when you add an index kind, and a kind that forgets this file
+ * fails a test instead of silently reporting the previous era's answer — which
+ * is exactly what #1344 and #1345 did to the pre-#1344 `explain()`.
+ *
+ * ⚠️ The observation is "did the executor READ RECORDS OUT OF the snapshot",
+ * not "did it CALL `snapshot()`". #1344's `orderedIndexRows` and #1345's
+ * `compoundCandidates` both call `snapshot()` for their coverage check and
+ * read only its `.length`, so the original call-counter witness would now
+ * report a scan for two paths that never touch a record.
+ */
+import type { QueryPlan } from '../builder.js';
+import type { JoinContext } from './join.js';
+import type { ViaPipeline } from '../../via/pipeline.js';
+/**
+ * How a node is executed. An OPEN label set on purpose — a consumer reads
+ * these as strings, so a build that knows a dispatch this one does not still
+ * type-checks. A new index kind adds its label here, its branch in
+ * {@link indexDispatchFor} (or {@link orderedIndexPick} when it replaces the
+ * `sort` node's dispatch), AND a row in the witness table named in this file's
+ * header. No consumer shape moves.
+ */
+export type ExplainDispatch = 'source' | 'index:hash'
+/** #1344 — a sorted index served `<` `<=` `>` `>=` `between` `startsWith`. */
+ | 'index:range'
+/** #1345 — a compound tuple served an equality prefix (+ an optional range). */
+ | 'index:compound'
+/**
+ * #1355 — a sorted index served a UNION of `startsWith` slices named by a
+ * Via binding's prefix probe (`geo()`'s `near`). Unlike every other index
+ * label, the clause is NOT consumed: the cover is a superset and the
+ * binding's predicate still runs over the candidates, so `estimatedRows`
+ * here is the candidate count, not the answer's.
+ */
+ | 'index:prefix'
+/**
+ * #1344 / #1345 — an ordered index walk answered `orderBy(f).limit(n)`
+ * outright: no sort runs, and only the page's worth of records is read.
+ */
+ | 'index:ordered' | 'scan' | 'join:nested' | 'join:hash' | 'join:reverse-index'
+/** #1339 — a declared composite `on`, hashed over a tuple key. */
+ | 'join:composite-hash'
+/** #1339 — a declared range `on`, nested-loop over a sorted right side. */
+ | 'join:sorted-range' | 'crossJoin' | 'sort' | 'page'
+/**
+ * #1342 — a partitioned union read every declared partition, because the
+ * predicate proved no narrowing. The SOUND default: everything that is not
+ * on `resolvePartitionScope`'s whitelist lands here.
+ */
+ | 'partitions:all'
+/**
+ * #1342 — a top-level AND-ed `==`/`in` on the declared partition key
+ * narrowed the set, so the excluded members were never asked for.
+ *
+ * ⚠️ Unlike every other label in this union, this one is NOT decided in
+ * this file. `resolvePartitionScope` (`query/partition.ts`) is called by
+ * the executor and by `explain()`, so there is nothing to keep in step —
+ * see that module's header for why a third mirror site was refused.
+ */
+ | 'partitions:pruned' | (string & {});
+/** One line of the plan. */
+export interface ExplainNode {
+    /** Pipeline stage: 'source' | 'where' | 'group' | 'filter' | 'wherePredicate' | 'crossJoin' | 'join' | 'orderBy' | 'page'. */
+    readonly op: string;
+    /** How this stage runs. */
+    readonly dispatch: ExplainDispatch;
+    /** One-line, no-op-prefix description, e.g. `status == "open"`. */
+    readonly detail: string;
+    /**
+     * Rows this stage is expected to emit. Exact where an index or a snapshot
+     * size supplies it; an upper bound (carried through unchanged) where only a
+     * scan can answer — the accompanying note says which.
+     */
+    readonly estimatedRows: number | undefined;
+    /** Why this dispatch, and anything about to trip. */
+    readonly notes: readonly string[];
+    /** Sub-clauses of an `and`/`or` group. Empty for every other node. */
+    readonly children: readonly ExplainNode[];
+}
+/** A cardinality ceiling this plan runs against. */
+export interface ExplainCap {
+    /** e.g. `join:client:right`, `crossJoin:lines`. */
+    readonly name: string;
+    /** The ceiling that would throw. */
+    readonly limit: number;
+    /** The size measured (or estimated) against it. */
+    readonly observed: number;
+    /** `warn` mirrors the executor's 80%-of-ceiling warning; `exceeded` would throw. */
+    readonly status: 'ok' | 'warn' | 'exceeded';
+}
+/** The result of {@link explainPlan}. */
+export interface QueryExplanation {
+    /** The plan tree, in execution order. */
+    readonly nodes: readonly ExplainNode[];
+    /** Every cardinality ceiling this plan runs against. */
+    readonly caps: readonly ExplainCap[];
+    /**
+     * Fields this query references whose Via binding rewrites aggregate
+     * reducers — money's exact-BigInt sum being the one that ships. Applies to
+     * `.aggregate()`/`.groupBy()` over these fields, not to the row pipeline.
+     */
+    readonly reducerRewrite: readonly {
+        readonly brand: string;
+        readonly field: string;
+    }[];
+    /** One line per node, depth-indented. Stable for a given plan. */
+    readonly text: string;
+}
+/**
+ * The slice of a query source `explain()` reads. Structurally satisfied by
+ * the builder's internal source; declared here so this module does not
+ * depend on the builder's non-exported types.
+ */
+export interface ExplainSource {
+    snapshot(): readonly unknown[];
+    /**
+     * #1414/#1421 — the cold-collection gate. `explain()` is the one terminal
+     * that stays answerable while unhydrated (a diagnostic that refuses to
+     * diagnose is useless), so it carries the distinction instead: a snapshot
+     * of zero rows because nothing has been loaded is NOT the same fact as a
+     * snapshot of zero rows because the collection is empty, and only the
+     * second one is safe to act on.
+     */
+    hydration?: {
+        isHydrated(): boolean;
+    };
+    getIndexes?(): ExplainIndexProbe | null;
+    lookupById?(id: string): unknown;
+    via?: ViaPipeline;
+}
+/**
+ * The index-store surface `explain()` probes. Declared structurally rather
+ * than importing `CollectionIndexes`: the kernel spine may not statically
+ * reach a `with-*` service (`check:architecture`'s port-layering gate), and
+ * every method here is a pure read. `CollectionIndexes` satisfies it.
+ */
+export interface ExplainIndexProbe {
+    has(field: string): boolean;
+    fields(): string[];
+    lookupEqual(field: string, value: unknown): ReadonlySet<string> | null;
+    lookupIn(field: string, values: readonly unknown[]): ReadonlySet<string> | null;
+    /** #1344 — does a SORTED index cover this field? */
+    hasSorted?(field: string): boolean;
+    /** #1344 — entry count of the sorted index; compare against the snapshot. */
+    sortedSize?(field: string): number;
+    /** #1344 — ids satisfying a range operator, or `null` when unserved. */
+    lookupRange?(field: string, op: string, value: unknown): ReadonlySet<string> | null;
+    /** #1344 — every indexed id in field order, or `null` when unserved. */
+    orderedIds?(field: string, direction: 'asc' | 'desc'): readonly string[] | null;
+    /** #1345 — declared field tuples, in declaration order. */
+    compoundTuples?(): ReadonlyArray<readonly string[]>;
+    /** #1345 — records the tuple's index holds; compare against the snapshot. */
+    compoundSize?(fields: readonly string[]): number;
+    /** #1345 — ids for an equality prefix, optionally narrowed by a range. */
+    lookupCompound?(fields: readonly string[], prefixValues: readonly unknown[], range?: {
+        readonly op: string;
+        readonly value: unknown;
+    }): ReadonlySet<string> | null;
+    /** #1345 — an equality prefix ordered by the remaining component. */
+    compoundOrderedIds?(fields: readonly string[], prefixValues: readonly unknown[], direction: 'asc' | 'desc'): readonly string[] | null;
+}
+/** Build the explanation for a plan. Reads sizes and index buckets; runs nothing. */
+export declare function explainPlan(source: ExplainSource, plan: QueryPlan, joinContext?: JoinContext): QueryExplanation;
+/**
+ * One line per node, two spaces per level of depth.
+ *
+ * Exported (#1342) so the partitioned union's `explain()` renders its
+ * partition line and the member's nodes with THIS renderer rather than a
+ * second one — a consumer diffing two `explain().text` outputs must not be
+ * reading two formatters.
+ */
+export declare function renderExplainText(nodes: readonly ExplainNode[]): string;

package/dist/kernel/query/relate/index.d.ts

@@ -0,0 +1,48 @@
+/**
+ * `@noy-db/hub/query/relate` — **the Relate group of the query DSL.**
+ *
+ * ```ts
+ * import '@noy-db/hub/query/relate'   // once, in your app's entry
+ *
+ * invoices.query()
+ *   .where('status', '==', 'paid')
+ *   .join<'client', Client>('clientId', { as: 'client' })
+ *   .toArray()
+ * ```
+ *
+ * ⭐ **THIS MODULE EXISTS FOR ITS SIDE EFFECT.** Importing it patches
+ * `Query.prototype` with the ten Relate methods and fills the executor's join
+ * hooks; the `declare module` block below merges the same methods into the
+ * `Query` TYPE, so the method and its signature arrive together and a call
+ * without this import does not compile.
+ *
+ * ⚠️ It is listed in `package.json`'s `sideEffects` array for exactly that
+ * reason. A bundler told `sideEffects: false` about this file would drop the
+ * import — and the failure is a runtime `QueryExtensionMissingError` in
+ * production from code that typechecked. Do not remove it from that list.
+ *
+ * The value exports below are the free functions Relate has always published
+ * through `@noy-db/hub/query`; they moved here with their group.
+ */
+import type { RelateSurface } from './methods.js';
+import type { ScanRelateSurface } from './scan-methods.js';
+declare module '../builder.js' {
+    interface Query<T, S extends keyof T = never, Q extends keyof T & string = never, M extends keyof T & string = never> extends RelateSurface<T, S, Q, M> {
+    }
+}
+declare module '../scan-builder.js' {
+    interface ScanBuilder<T, S extends keyof T = never, M extends keyof T & string = never> extends ScanRelateSurface<T, S, M> {
+    }
+}
+export { applyJoins, DEFAULT_JOIN_MAX_ROWS, DEFAULT_CROSS_JOIN_MAX_ROWS, resetJoinWarnings } from './join.js';
+export type { JoinLeg, JoinContext, JoinableSource, JoinStrategy, JoinDirection } from './join.js';
+export type { JoinOnSpec, JoinOnPlan, JoinOnOp } from './join-on.js';
+export { runTraversal } from './traverse.js';
+export type { TraversalRow, TraverseOptions, TraverseDirection, TraverseSource, CyclePolicy } from './traverse.js';
+export { explainPlan, renderExplainText } from './explain.js';
+export type { QueryExplanation, ExplainNode, ExplainCap, ExplainDispatch, ExplainSource, ExplainIndexProbe } from './explain.js';
+export { ALL_PARTITIONS, describePartitionScope, partitionsInScope, resolvePartitionScope, } from './partition.js';
+export type { PartitionScope } from './partition.js';
+export type { QueryPlan } from '../builder.js';
+export type { Clause } from '../predicate.js';
+export { JoinTooLargeError, DanglingReferenceError, TraversalCycleError, CrossJoinTooLargeError, CrossJoinSourceUnknownError, } from '../../errors.js';

package/dist/kernel/query/relate/install.d.ts

@@ -0,0 +1,2 @@
+/** Idempotent: the root barrel and the subpath entry both call it. */
+export declare function installRelate(): void;

package/dist/kernel/query/relate/join-on.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Declared non-equi and composite joins — `.joinOn(target, { on })` (#1339).
+ *
+ * `join.ts` resolves a `ref()`-declared FK against the target's `id`. That is
+ * one shape of join. This module is the other two the query DSL could not
+ * express at all:
+ *
+ *   - **composite equality** — `on: [['clientId','clientId'], ['year','year']]`
+ *   - **range / theta**      — `on: { left: 'date', op: 'between', right: ['from','to'] }`
+ *
+ * ⭐ THE DECLARED FORM IS THE WHOLE POINT, and it is the only reason this is
+ * not `crossJoin({ on: fn })` with extra steps. `crossJoin`'s `on` is a
+ * closure: it pairs arbitrary rows perfectly and serializes to the sentinel
+ * `'[inline]'`, which is why a materialized view over one has its drift
+ * detection disabled. A `JoinOnPlan` is plain JSON, so `serializePlan()`
+ * emits it verbatim, `summarizeQueryPlan()` folds it into the MV
+ * `queryHash`, and two plans differing only in their `on` are two different
+ * queries — pinned by `__tests__/query-join-on.test.ts`.
+ *
+ * ⛔ THE NORMAL FORM IS PART OF THE HASH. `normalizeJoinOn` is the ONLY place
+ * a `JoinOnSpec` becomes a `JoinOnPlan`, and it sorts composite pairs so two
+ * logically identical joins hash identically (an AND of equalities is
+ * commutative). Changing the normal form moves every stored `queryHash` for a
+ * `joinOn` MV. Adding a new `on` KIND does not, because the leg is omitted
+ * entirely when absent — the same discipline `direction` (#1289) and `inner`
+ * (#1361) follow.
+ *
+ * **Key encoding.** Both strategies key on {@link SortKey} — the per-component
+ * ordered key #1345's `CompoundIndex` already uses — rather than a second,
+ * hand-rolled encoding. That buys the kind-ranking invariant for free: the
+ * string `'2026'` and the number `2026` live in different kinds and never
+ * match, exactly as `predicate.ts`'s `isComparable` refuses to compare them.
+ * A component whose value has no order-defined type (nullish, boolean,
+ * object, array) makes the whole record unkeyable and it drops out of the
+ * probe — same rule as `CompoundIndex`, where a record is indexed only when
+ * EVERY component has an ordered value.
+ *
+ * **Cost, stated honestly:**
+ *
+ * | strategy | build | per left row | total |
+ * |---|---|---|---|
+ * | `join:composite-hash` | O(m) one pass | O(1) + matches | **O(n + m + output)** |
+ * | `join:sorted-range` (`< <= > >=`) | O(m log m) one sort | O(log m) + matches | **O(m log m + n log m + output)** |
+ * | `join:sorted-range` (`between`) | O(m log m) | O(log m + p), p = rights whose `from` ≤ probe | **O(m log m + n·p)** — degrades to O(n·m) when every interval starts early |
+ *
+ * ⚠️ Neither is index-aware in the `lookupRange`/`lookupCompound` sense: a
+ * `JoinableSource` exposes `snapshot()` and `lookupById()` and nothing else,
+ * so the right collection's declared indexes are not reachable from here.
+ * The sort/hash is built per execution over the right SNAPSHOT. Reaching a
+ * real index would mean widening the published `JoinableSource` seam and
+ * taking a kernel→`with-lookup` runtime edge for a service that is opt-in;
+ * that is a separate decision, not a detail of this one.
+ *
+ * ⚠️ **A theta join EXPANDS rows**, which the ref join never does — so the
+ * per-side `maxRows` ceilings are not sufficient on their own. The OUTPUT is
+ * checked against the same ceiling as it is produced, and the join throws
+ * `JoinTooLargeError` with `side: 'output'` rather than allocating an
+ * unbounded cartesian product. That check is the difference between an error
+ * and a hang.
+ */
+/** Comparison operators a declared range join understands. */
+export type JoinOnOp = '<' | '<=' | '>' | '>=' | 'between';
+/**
+ * The consumer-facing `on`, in either of the two shapes #1339 proposed.
+ *
+ *   - `[['clientId','clientId'], ['year','year']]` — composite equality; every
+ *     pair must match. Each pair is `[leftField, rightField]`.
+ *   - `{ left, op, right }` — a comparison of ONE left field against one right
+ *     field (`< <= > >=`), or against a right INTERVAL for `between`, where
+ *     `right` names the two boundary fields `[fromField, toField]` and the
+ *     bounds are inclusive.
+ */
+export type JoinOnSpec = readonly (readonly [string, string])[] | {
+    readonly left: string;
+    readonly op: Exclude<JoinOnOp, 'between'>;
+    readonly right: string;
+} | {
+    readonly left: string;
+    readonly op: 'between';
+    readonly right: readonly [string, string];
+};
+/**
+ * The normalised, serialisable form stored on the {@link JoinLeg}. This is
+ * what lands in `toPlan()` and in the MV `queryHash` — see the module header
+ * before changing its shape.
+ */
+export type JoinOnPlan = {
+    readonly kind: 'composite';
+    /** `[leftField, rightField]` pairs, sorted by left then right field. */
+    readonly pairs: readonly (readonly [string, string])[];
+} | {
+    readonly kind: 'range';
+    readonly left: string;
+    readonly op: JoinOnOp;
+    /** One right field, or the two inclusive boundary fields for `between`. */
+    readonly right: readonly string[];
+};
+/**
+ * Validate and normalise a consumer `on` into its plan form.
+ *
+ * Every refusal is at PLAN time, with the offending shape named: an `on` that
+ * cannot be serialised deterministically must never reach a `queryHash`,
+ * because a hash over an invalid plan is a hash nothing can reproduce.
+ */
+export declare function normalizeJoinOn(on: JoinOnSpec, target: string): JoinOnPlan;
+/** A short, stable rendering of the predicate for `explain()`'s detail line. */
+export declare function describeJoinOn(on: JoinOnPlan): string;
+/** `explain()`'s dispatch label for a declared-`on` leg. */
+export declare function joinOnDispatch(on: JoinOnPlan): 'join:composite-hash' | 'join:sorted-range';
+/** One row the declared join produced: the left row and its matched right record. */
+export interface JoinOnMatch {
+    readonly left: unknown;
+    /** `undefined` when this left row matched nothing — the left-outer row. */
+    readonly right: unknown;
+}
+/**
+ * Run a declared `on` against the right snapshot, emitting one
+ * {@link JoinOnMatch} per matched pair plus one `right: undefined` match per
+ * left row that matched nothing (the caller applies ref-mode / inner-drop
+ * semantics, exactly as it does for the ref join).
+ *
+ * `onOverflow` is called the moment the produced row count would exceed the
+ * ceiling; it is expected to throw. Passing the check as a callback keeps the
+ * error's wording — which names the leg and the ceiling — with the leg.
+ */
+export declare function matchDeclaredJoin(leftRows: readonly unknown[], on: JoinOnPlan, rightSnapshot: readonly unknown[], maxRows: number, onOverflow: (produced: number) => never): JoinOnMatch[];

package/dist/kernel/query/relate/join-reduce.d.ts

@@ -0,0 +1,16 @@
+import { ViaPipeline } from '../../via/pipeline.js';
+/**
+ * The pipeline `aggregate()` / `groupBy()` should reduce THIS plan with: the
+ * source's own, plus the alias binding when the plan can produce an alias.
+ *
+ * Returns the left pipeline unchanged when there is no alias to serve, so a
+ * query with no joins — every query written before #1338 — is handed exactly
+ * the object it was handed before.
+ */
+export declare function reduceViaFor(leftVia: ViaPipeline | undefined, aliasVia: ReadonlyMap<string, ViaPipeline> | undefined): ViaPipeline | undefined;
+/**
+ * The pipeline for a grouped plan whose GROUP KEY stays on the left: the
+ * source's own, plus the refusal above so an aliased reducer is a loud error
+ * rather than a confident zero. No aliases in the plan → unchanged.
+ */
+export declare function refuseAliasReduceVia(leftVia: ViaPipeline | undefined, aliases: ReadonlySet<string>): ViaPipeline | undefined;