@noy-db/hub 0.1.0-pre.14 → 0.1.0-pre.16

package/dist/indexing/index.js

@@ -7,12 +7,12 @@ import {
   decodeIdxId,
   encodeIdxId,
   isIdxId
-} from "../chunk-PNTWORBA.js";
+} from "../chunk-DYECX3IX.js";
 import {
   CollectionIndexes
 } from "../chunk-YMYK7US4.js";
 import "../chunk-MRIBLZL3.js";
-import "../chunk-W7M3E5M5.js";
+import "../chunk-ADQ5MQ54.js";
 
 // src/indexing/active.ts
 function withIndexing() {

package/dist/{ledger-LHLIVPJ3.js → ledger-GHIZQLKR.js}

@@ -5,7 +5,7 @@ import {
   LedgerStore,
   applyPatch,
   computePatch
-} from "./chunk-E3VAKWJ7.js";
+} from "./chunk-QQUUAXDY.js";
 import {
   canonicalJson,
   envelopePayloadHash,
@@ -13,10 +13,10 @@ import {
   paddedIndex,
   parseIndex,
   sha256Hex
-} from "./chunk-WLZWITX3.js";
-import "./chunk-PMMNH7VZ.js";
-import "./chunk-7AQKLLI5.js";
-import "./chunk-W7M3E5M5.js";
+} from "./chunk-2AXFIYHT.js";
+import "./chunk-H7XROQJM.js";
+import "./chunk-WCA2NROQ.js";
+import "./chunk-ADQ5MQ54.js";
 export {
   LEDGER_COLLECTION,
   LEDGER_DELTAS_COLLECTION,
@@ -30,4 +30,4 @@ export {
   parseIndex,
   sha256Hex
 };
-//# sourceMappingURL=ledger-LHLIVPJ3.js.map
+//# sourceMappingURL=ledger-GHIZQLKR.js.map

package/dist/materialized-views/index.cjs

@@ -21,7 +21,7 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/errors.ts
-var NoydbError, ValidationError, MaterializedViewCycleError, MaterializedViewSourceUnknownError, MaterializedViewTooLargeError;
+var NoydbError, ValidationError, GroupCardinalityError, MaterializedViewCycleError, MaterializedViewSourceUnknownError, MaterializedViewTooLargeError, MaterializedViewConfigError;
 var init_errors = __esm({
   "src/errors.ts"() {
     "use strict";
@@ -40,6 +40,24 @@ var init_errors = __esm({
         this.name = "ValidationError";
       }
     };
+    GroupCardinalityError = class extends NoydbError {
+      /** The field being grouped on. */
+      field;
+      /** Observed number of distinct groups at the moment the cap tripped. */
+      cardinality;
+      /** The cap that was exceeded. */
+      maxGroups;
+      constructor(field, cardinality, maxGroups) {
+        super(
+          "GROUP_CARDINALITY",
+          `.groupBy("${field}") produced ${cardinality} distinct groups, exceeding the ${maxGroups}-group ceiling. This is almost always a query mistake \u2014 grouping on a high-uniqueness field like "id" or "createdAt" produces one bucket per record. Narrow the query with .where() before grouping, or group on a lower-cardinality field (status, category, clientId). If you genuinely need high-cardinality grouping, file an issue with your use case.`
+        );
+        this.name = "GroupCardinalityError";
+        this.field = field;
+        this.cardinality = cardinality;
+        this.maxGroups = maxGroups;
+      }
+    };
     MaterializedViewCycleError = class extends NoydbError {
       path;
       constructor(path) {
@@ -79,6 +97,15 @@ var init_errors = __esm({
         this.limit = limit;
       }
     };
+    MaterializedViewConfigError = class extends NoydbError {
+      constructor(message) {
+        super(
+          "MATERIALIZED_VIEW_CONFIG",
+          `[noy-db] withMaterializedView: ${message}`
+        );
+        this.name = "MaterializedViewConfigError";
+      }
+    };
   }
 });
 
@@ -115,6 +142,12 @@ function summarizeQueryPlan(query) {
     joins: plan.joins.map((j) => ({ field: j.field, as: j.as, target: j.target, mode: j.mode }))
   });
 }
+function summarizeUnionPlan(spec) {
+  const arms = (spec.unionSources ?? []).map((s) => s.collection).join(",");
+  const groupBy = Array.isArray(spec.groupBy) ? [...spec.groupBy].sort().join(",") : typeof spec.groupBy === "string" ? spec.groupBy : "";
+  const aggKeys = spec.aggregate ? Object.keys(spec.aggregate).sort().join(",") : "";
+  return `union(${arms})|groupBy(${groupBy})|aggregate(${aggKeys})`;
+}
 var init_dependency_analyzer = __esm({
   "src/materialized-views/dependency-analyzer.ts"() {
     "use strict";
@@ -241,27 +274,34 @@ var init_registry = __esm({
        */
       async register(spec, db, options) {
         const dbForQuery = spec.predicates ? wrapDbWithPredicates(db, spec.predicates) : db;
-        const q = spec.query(dbForQuery);
-        const qAny = q;
-        const isQuery = typeof qAny._plan === "function";
         let dependencies;
         let queryPlanSummary;
-        if (isQuery) {
-          dependencies = analyzeDependencies(q);
-          queryPlanSummary = summarizeQueryPlan(q);
-          const predicateRefs = extractPredicateRefs(qAny._plan());
-          if (predicateRefs.length > 0) {
-            queryPlanSummary = JSON.stringify({ plan: queryPlanSummary, predicates: predicateRefs });
-          }
-          if (spec.sources) for (const s of spec.sources) dependencies.add(s);
+        let qAny = null;
+        let isQuery = false;
+        if (spec.unionSources) {
+          dependencies = new Set(spec.unionSources.map((s) => s.collection));
+          queryPlanSummary = summarizeUnionPlan(spec);
         } else {
-          if (!spec.sources || spec.sources.length === 0) {
-            throw new Error(
-              `withMaterializedView "${spec.name}": query() returned an aggregate (Aggregation or GroupedAggregation) but no \`sources\` field is declared. The dependency analyzer cannot walk through groupBy().aggregate() back to the source \u2014 declare sources: [...] explicitly.`
-            );
+          const q = spec.query(dbForQuery);
+          qAny = q;
+          isQuery = typeof qAny._plan === "function";
+          if (isQuery) {
+            dependencies = analyzeDependencies(q);
+            queryPlanSummary = summarizeQueryPlan(q);
+            const predicateRefs = extractPredicateRefs(qAny._plan());
+            if (predicateRefs.length > 0) {
+              queryPlanSummary = JSON.stringify({ plan: queryPlanSummary, predicates: predicateRefs });
+            }
+            if (spec.sources) for (const s of spec.sources) dependencies.add(s);
+          } else {
+            if (!spec.sources || spec.sources.length === 0) {
+              throw new Error(
+                `withMaterializedView "${spec.name}": query() returned an aggregate (Aggregation or GroupedAggregation) but no \`sources\` field is declared. The dependency analyzer cannot walk through groupBy().aggregate() back to the source \u2014 declare sources: [...] explicitly.`
+              );
+            }
+            dependencies = new Set(spec.sources);
+            queryPlanSummary = JSON.stringify({ aggregate: true, sources: [...spec.sources].sort() });
           }
-          dependencies = new Set(spec.sources);
-          queryPlanSummary = JSON.stringify({ aggregate: true, sources: [...spec.sources].sort() });
         }
         if (options?.knownCollections) {
           for (const dep of dependencies) {
@@ -370,6 +410,119 @@ var init_registry = __esm({
   }
 });
 
+// src/query/predicate.ts
+function readPath(record, path) {
+  if (record === null || record === void 0) return void 0;
+  if (!path.includes(".")) {
+    return record[path];
+  }
+  const segments = path.split(".");
+  let cursor = record;
+  for (const segment of segments) {
+    if (cursor === null || cursor === void 0) return void 0;
+    cursor = cursor[segment];
+  }
+  return cursor;
+}
+var init_predicate = __esm({
+  "src/query/predicate.ts"() {
+    "use strict";
+  }
+});
+
+// src/aggregate/canonical-key.ts
+function canonicalGroupKey(fields, row) {
+  const sorted = [...fields].sort();
+  const parts = [];
+  for (const name of sorted) {
+    const v = row[name];
+    const serialised = v === void 0 ? "undefined" : JSON.stringify(v);
+    parts.push(`${name}=${serialised}`);
+  }
+  return parts.join("|");
+}
+var init_canonical_key = __esm({
+  "src/aggregate/canonical-key.ts"() {
+    "use strict";
+  }
+});
+
+// src/aggregate/groupby.ts
+function warnCardinalityApproaching(fields, observed) {
+  const key = JSON.stringify([...fields].sort());
+  if (warnedCardinalityFields.has(key)) return;
+  warnedCardinalityFields.add(key);
+  const label = `[${fields.join(", ")}]`;
+  console.warn(
+    `[noy-db] .groupBy(${label}) produced ${observed} distinct groups, ${Math.round(observed / GROUPBY_MAX_CARDINALITY * 100)}% of the ${GROUPBY_MAX_CARDINALITY}-group ceiling. Narrow the query with .where() before grouping, or switch to a lower-cardinality field.`
+  );
+}
+function groupAndReduce(records, fieldOrFields, spec) {
+  const fields = typeof fieldOrFields === "string" ? [fieldOrFields] : fieldOrFields;
+  if (fields.length === 0) {
+    throw new Error(".groupBy() requires at least one field");
+  }
+  const buckets = /* @__PURE__ */ new Map();
+  const fieldLabel = fields.length === 1 ? fields[0] : `[${fields.join(", ")}]`;
+  for (const record of records) {
+    const keyValues = {};
+    for (const f of fields) {
+      keyValues[f] = readPath(record, f);
+    }
+    const dedupKey = canonicalGroupKey(fields, keyValues);
+    let bucket = buckets.get(dedupKey);
+    if (bucket === void 0) {
+      if (buckets.size >= GROUPBY_MAX_CARDINALITY) {
+        throw new GroupCardinalityError(
+          fieldLabel,
+          buckets.size + 1,
+          GROUPBY_MAX_CARDINALITY
+        );
+      }
+      bucket = { keyValues, records: [] };
+      buckets.set(dedupKey, bucket);
+    }
+    bucket.records.push(record);
+  }
+  if (buckets.size >= GROUPBY_WARN_CARDINALITY) {
+    warnCardinalityApproaching(fields, buckets.size);
+  }
+  const reducerKeys = Object.keys(spec);
+  const out = [];
+  for (const bucket of buckets.values()) {
+    const state = {};
+    for (const rk of reducerKeys) {
+      state[rk] = spec[rk].init();
+    }
+    for (const record of bucket.records) {
+      for (const rk of reducerKeys) {
+        state[rk] = spec[rk].step(state[rk], record);
+      }
+    }
+    const row = {};
+    for (const f of fields) {
+      row[f] = bucket.keyValues[f];
+    }
+    for (const rk of reducerKeys) {
+      row[rk] = spec[rk].finalize(state[rk]);
+    }
+    out.push(row);
+  }
+  return out;
+}
+var GROUPBY_WARN_CARDINALITY, GROUPBY_MAX_CARDINALITY, warnedCardinalityFields;
+var init_groupby = __esm({
+  "src/aggregate/groupby.ts"() {
+    "use strict";
+    init_predicate();
+    init_canonical_key();
+    init_errors();
+    GROUPBY_WARN_CARDINALITY = 1e4;
+    GROUPBY_MAX_CARDINALITY = 1e5;
+    warnedCardinalityFields = /* @__PURE__ */ new Set();
+  }
+});
+
 // src/materialized-views/executor.ts
 var executor_exports = {};
 __export(executor_exports, {
@@ -390,6 +543,27 @@ async function materializeQueryResult(q, mvName) {
     `MV "${mvName}": query() must return a Query<T>, Aggregation, or GroupedAggregation. Got something without a .toArray() or .run() terminal.`
   );
 }
+async function materializeUnionResult(spec, db) {
+  const unified = [];
+  for (const arm of spec.unionSources) {
+    const coll = db.collection(arm.collection);
+    const sourceRows = coll.query().toArray();
+    for (const r of sourceRows) {
+      unified.push(arm.map(r));
+    }
+  }
+  if (!spec.groupBy) return unified;
+  const groupFields = typeof spec.groupBy === "string" ? [spec.groupBy] : spec.groupBy;
+  if (!spec.aggregate) {
+    const seen = /* @__PURE__ */ new Map();
+    for (const row of unified) {
+      const k = canonicalGroupKey(groupFields, row);
+      if (!seen.has(k)) seen.set(k, row);
+    }
+    return [...seen.values()];
+  }
+  return groupAndReduce(unified, groupFields, spec.aggregate);
+}
 async function listOutputIds(outputColl) {
   const cAny = outputColl;
   const adapter = cAny.adapter;
@@ -409,6 +583,8 @@ var init_executor = __esm({
     "use strict";
     init_errors();
     init_registry();
+    init_groupby();
+    init_canonical_key();
     DEFAULT_MAX_ROWS = 1e5;
     MaterializedViewExecutor = {
       async refresh(reg, accessor) {
@@ -419,8 +595,13 @@ var init_executor = __esm({
         const strict = spec.strict ?? false;
         const baseCtx = accessor.getQueryContext();
         const ctxForQuery = spec.predicates ? wrapDbWithPredicates(baseCtx, spec.predicates) : baseCtx;
-        const q = spec.query(ctxForQuery);
-        const rows = await materializeQueryResult(q, spec.name);
+        let rows;
+        if (spec.unionSources) {
+          rows = await materializeUnionResult(spec, ctxForQuery);
+        } else {
+          const q = spec.query(ctxForQuery);
+          rows = await materializeQueryResult(q, spec.name);
+        }
         if (rows.length > maxRows) {
           throw new MaterializedViewTooLargeError(spec.name, rows.length, maxRows);
         }
@@ -489,6 +670,7 @@ var init_executor = __esm({
 // src/materialized-views/index.ts
 var materialized_views_exports = {};
 __export(materialized_views_exports, {
+  MaterializedViewConfigError: () => MaterializedViewConfigError,
   MaterializedViewCycleError: () => MaterializedViewCycleError,
   MaterializedViewExecutor: () => MaterializedViewExecutor,
   MaterializedViewRegistry: () => MaterializedViewRegistry,
@@ -512,9 +694,60 @@ function withMaterializedView(spec) {
   if (!spec.name || spec.name.length === 0) {
     throw new ValidationError("withMaterializedView: name is required");
   }
-  if (typeof spec.query !== "function") {
+  if (spec.query && spec.unionSources) {
+    throw new MaterializedViewConfigError(
+      "query and unionSources are mutually exclusive \u2014 pick one"
+    );
+  }
+  if (!spec.query && !spec.unionSources) {
+    throw new MaterializedViewConfigError(
+      "strategy must declare either query or unionSources"
+    );
+  }
+  if (spec.query !== void 0 && typeof spec.query !== "function") {
     throw new ValidationError("withMaterializedView: query must be a function returning a Query<T>");
   }
+  if (spec.unionSources) {
+    if (spec.unionSources.length < 2) {
+      throw new MaterializedViewConfigError(
+        "unionSources requires at least 2 source collections"
+      );
+    }
+    const seen = /* @__PURE__ */ new Set();
+    for (const s of spec.unionSources) {
+      if (typeof s?.collection !== "string" || s.collection.length === 0) {
+        throw new MaterializedViewConfigError(
+          "each unionSources entry must declare a non-empty `collection` string"
+        );
+      }
+      if (typeof s.map !== "function") {
+        throw new MaterializedViewConfigError(
+          `unionSources entry for "${s.collection}" is missing a \`map\` function`
+        );
+      }
+      if (seen.has(s.collection)) {
+        throw new MaterializedViewConfigError(
+          `unionSources must reference distinct collections (duplicate: "${s.collection}")`
+        );
+      }
+      seen.add(s.collection);
+    }
+    if (Array.isArray(spec.groupBy) && spec.groupBy.length === 0) {
+      throw new MaterializedViewConfigError(
+        `withMaterializedView "${spec.name}": groupBy must not be an empty array \u2014 omit it or provide at least one field name`
+      );
+    }
+    if (spec.aggregate && !spec.groupBy) {
+      throw new MaterializedViewConfigError(
+        `withMaterializedView "${spec.name}": UNION strategy with aggregate requires groupBy \u2014 use groupBy to declare the bucketing keys, or remove aggregate for a pure dedup MV`
+      );
+    }
+    if (spec.predicates) {
+      throw new MaterializedViewConfigError(
+        `withMaterializedView "${spec.name}": predicates are not supported on UNION strategies \u2014 UNION mode does not use a Query<T> chain, so .wherePredicate() cannot fire. Use the query() form, or open an issue if per-arm predicates are needed`
+      );
+    }
+  }
   if (typeof spec.rowKey !== "function") {
     throw new ValidationError("withMaterializedView: rowKey is required (no default; see spec \xA7 Type surface)");
   }
@@ -585,6 +818,7 @@ function clearMVStale(registry, mvName) {
 init_errors();
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  MaterializedViewConfigError,
   MaterializedViewCycleError,
   MaterializedViewExecutor,
   MaterializedViewRegistry,