@minnowdb/core 0.7.1 → 0.7.2

package/dist/engine/database.js

@@ -5373,13 +5373,14 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     }
     const open = this.#openTransaction;
     if (open !== void 0) {
-      const boundStatement = bindStatementParameters(statement, params);
-      if (!isTransactionalStatement(boundStatement)) {
-        throw new TypeError(`${boundStatement.kind.toUpperCase().replace("-", " ")} is not allowed inside a transaction`);
+      const boundStatement2 = bindStatementParameters(statement, params);
+      if (!isTransactionalStatement(boundStatement2)) {
+        throw new TypeError(`${boundStatement2.kind.toUpperCase().replace("-", " ")} is not allowed inside a transaction`);
       }
-      return externalizeExecuteResult(await this.#projectReturningItems(boundStatement, await this.#duringTransaction(open, () => open.session.executeStatement(boundStatement))));
+      return externalizeExecuteResult(await this.#projectReturningItems(boundStatement2, await this.#duringTransaction(open, () => open.session.executeStatement(boundStatement2))));
     }
-    return externalizeExecuteResult(await this.#projectReturningItems(statement, await this.runStatement(bindStatementParameters(statement, params))));
+    const boundStatement = bindStatementParameters(statement, params);
+    return externalizeExecuteResult(await this.#projectReturningItems(boundStatement, await this.runStatement(boundStatement)));
   }
   async #assertForeignKeysPresent(table, rowsByColumn, read, transaction) {
     for (const key of table.foreignKeys ?? []) {

package/dist/engine/optimizer.js

@@ -1494,14 +1494,17 @@ function scalarOrderExpression(expression, rows) {
   const matches = rows.select.filter((item) => item.alias === expression.reference);
   return matches.length === 1 ? structuredClone(matches[0]?.expression ?? expression) : structuredClone(expression);
 }
-function emptyScalarExpression(expression) {
+function emptyScalarExpression(expression, preservedColumns = /* @__PURE__ */ new Set()) {
   if (isAggregateCall(expression)) {
     return { kind: "literal", value: expression.name === "COUNT" ? 0 : null };
   }
-  if (expression.kind === "column" || expression.kind === "wildcard") {
+  if (expression.kind === "column") {
+    return preservedColumns.has(expression.reference) ? structuredClone(expression) : { kind: "literal", value: null };
+  }
+  if (expression.kind === "wildcard") {
     return { kind: "literal", value: null };
   }
-  return mapChildExpressions(expression, emptyScalarExpression);
+  return mapChildExpressions(expression, (child) => emptyScalarExpression(child, preservedColumns));
 }
 function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
   const shape = scalarShape(inner);
@@ -1512,14 +1515,44 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
   if (rows.limitWithTies === true && rows.orderBy.length === 0) {
     throw new TypeError("A correlated scalar WITH TIES query requires ORDER BY");
   }
-  const keys = extractCorrelation(rows, scope, "scalar", true);
-  const outerExpressions = keys.map((key) => structuredClone(key.outer));
+  const expressionOutside = [];
+  collectExpressionOutsideReferences(shape.expression, /* @__PURE__ */ new Set([rows.base.alias, ...rows.joins.map((join) => join.alias)]), expressionOutside);
+  const probeLifted = expressionOutside.length > 0 || !canExtractCorrelation(rows, scope, "scalar", true);
+  const references = probeLifted ? probeOuterReferences(rows, expressionOutside, scope, "scalar") : [];
+  const keys = probeLifted ? takeCorrelationPredicates(rows, scope, true) : extractCorrelation(rows, scope, "scalar", true);
+  const outerExpressions = probeLifted ? references.map((reference) => ({ kind: "column", reference })) : keys.map((key) => structuredClone(key.outer));
+  const originalExpression = structuredClone(shape.expression);
   const probes = outerProbeBlockForExpressions(block, outerExpressions, nextAlias.resolvedNames);
   const probesAlias = nextAlias();
   const probeReferences = outerExpressions.map((_, index) => ({
     kind: "column",
     reference: `${probesAlias}.${correlationProbeAlias(index)}`
   }));
+  if (probeLifted) {
+    const replacements = new Map(references.map((reference, index) => [reference, probeReferences[index] ?? null]));
+    replacePlanReferences(rows, replacements);
+    shape.expression = replaceExpressionReferences(shape.expression, replacements);
+    const source = { table: probesAlias, alias: probesAlias, derived: probes };
+    const missingProbe = { kind: "literal", value: null };
+    const keyProbes = keys.map((key) => {
+      const reference = key.outer.kind === "column" ? key.outer.reference : "";
+      return probeReferences[references.indexOf(reference)] ?? missingProbe;
+    });
+    const keysReadBase = keys.every((key) => {
+      const parts = key.inner.kind === "column" ? key.inner.reference.split(".") : [];
+      return parts.length === 1 || parts[0] === rows.base.alias;
+    });
+    if (keys.length > 0 && keysReadBase) {
+      rows.joins.unshift(generalCorrelationJoin(probesAlias, probes, keys.map((key) => key.inner), keyProbes, keys.map((key) => key.operator)));
+    } else {
+      rows.joins.unshift(crossJoinPlan(source));
+      rows.predicates.push(...keys.map((key, index) => ({
+        left: key.inner,
+        operator: key.operator,
+        right: keyProbes[index] ?? missingProbe
+      })));
+    }
+  }
   const finalRowsAlias = nextAlias();
   const projected = [];
   const project = (expression, alias) => {
@@ -1556,6 +1589,15 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
       return rewritten;
     }
     if (expression.kind === "column" || expression.kind === "wildcard") {
+      if (probeLifted && expression.kind === "column") {
+        const index = probeReferences.findIndex((reference) => reference.kind === "column" && reference.reference === expression.reference);
+        if (index >= 0) {
+          return {
+            kind: "column",
+            reference: `${finalRowsAlias}.${correlationKeyAlias(index)}`
+          };
+        }
+      }
       throw new TypeError("A correlated scalar aggregate cannot select an ungrouped inner column");
     }
     if (expression.kind === "subquery" || expression.kind === "exists") {
@@ -1575,18 +1617,21 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
     base: rows.base,
     joins: rows.joins,
     select: [
-      ...keys.map((key, index) => ({
-        expression: key.inner,
+      ...(probeLifted ? probeReferences : keys.map((key) => key.inner)).map((expression, index) => ({
+        expression,
         alias: correlationKeyAlias(index)
       })),
       ...projected
     ],
-    predicates: [...rows.predicates, ...mirrorPredicatesThroughKeys(probes.predicates, keys)],
+    predicates: [
+      ...rows.predicates,
+      ...probeLifted ? [] : mirrorPredicatesThroughKeys(probes.predicates, keys)
+    ],
     groupBy: [],
     having: [],
     orderBy: []
   };
-  const matched = {
+  const matched = probeLifted ? rawRows : {
     sql: "(correlated scalar matched probes)",
     base: { table: probesAlias, alias: probesAlias, derived: probes },
     joins: [
@@ -1623,7 +1668,7 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
           {
             alias: correlationRowNumberAlias,
             name: rows.limitWithTies === true ? "RANK" : "ROW_NUMBER",
-            partitionAliases: keys.map((_, index) => correlationKeyAlias(index)),
+            partitionAliases: outerExpressions.map((_, index) => correlationKeyAlias(index)),
             orderAliases: rows.orderBy.map((order, index) => ({
               alias: correlationOrderAlias(index),
               direction: order.direction,
@@ -1659,7 +1704,7 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
       base: ranked,
       joins: [],
       select: [
-        ...keys.map((_, index) => ({
+        ...outerExpressions.map((_, index) => ({
           expression: {
             kind: "column",
             reference: `${rankedAlias}.${correlationKeyAlias(index)}`
@@ -1679,7 +1724,7 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
       ...rows.offsetParameter === void 0 ? {} : { offsetValidationParameters: [rows.offsetParameter] }
     };
   }
-  const finalProbeReferences = keys.map((_, index) => ({
+  const finalProbeReferences = outerExpressions.map((_, index) => ({
     kind: "column",
     reference: `${finalRowsAlias}.${correlationKeyAlias(index)}`
   }));
@@ -1716,7 +1761,7 @@ function decorrelateGeneralScalar(block, inner, scope, nextAlias) {
           left: marker,
           right: { kind: "literal", value: null }
         },
-        then: grouped ? emptyScalarExpression(structuredClone(shape.expression)) : { kind: "literal", value: null }
+        then: grouped ? emptyScalarExpression(originalExpression, new Set(references)) : { kind: "literal", value: null }
       }
     ],
     otherwise: {
@@ -1880,19 +1925,7 @@ function decorrelateExistsExpression(block, exists, scope, nextAlias) {
 }
 function decorrelateExistsWithProbes(block, exists, scope, nextAlias) {
   const inner = exists.block;
-  const outside = [];
-  collectOutsideReferences(inner, /* @__PURE__ */ new Set(), outside);
-  const references = [...new Set(outside)];
-  if (references.length === 0) {
-    throw new TypeError("Correlated EXISTS subquery has no usable outer reference");
-  }
-  for (const reference of references) {
-    const parts = reference.split(".");
-    const alias = parts.length === 2 ? parts[0] : void 0;
-    if (alias === void 0 || !scope.has(alias)) {
-      throw new TypeError(`Correlated EXISTS references an unavailable outer source: ${reference}`);
-    }
-  }
+  const references = probeOuterReferences(inner, [], scope, "EXISTS");
   const outerExpressions = references.map((reference) => ({
     kind: "column",
     reference
@@ -1959,27 +1992,30 @@ function decorrelateExistsWithProbes(block, exists, scope, nextAlias) {
     right: { kind: "literal", value: null }
   };
 }
-function replacePlanReferences(block, replacements) {
-  const rewrite = (expression) => {
-    if (expression.kind === "column") {
-      const replacement = replacements.get(expression.reference);
-      return replacement == null ? expression : structuredClone(replacement);
+function replaceExpressionReferences(expression, replacements) {
+  const rewrite = (expression2) => {
+    if (expression2.kind === "column") {
+      const replacement = replacements.get(expression2.reference);
+      return replacement == null ? expression2 : structuredClone(replacement);
     }
-    if (expression.kind === "subquery" || expression.kind === "exists") {
-      replacePlanReferences(expression.block, replacements);
-      return expression;
+    if (expression2.kind === "subquery" || expression2.kind === "exists") {
+      replacePlanReferences(expression2.block, replacements);
+      return expression2;
     }
-    if (expression.kind === "window") {
-      expression.partitionBy = expression.partitionBy.map(rewrite);
-      for (const order of expression.orderBy)
+    if (expression2.kind === "window") {
+      expression2.partitionBy = expression2.partitionBy.map(rewrite);
+      for (const order of expression2.orderBy)
         order.expression = rewrite(order.expression);
-      if (expression.argument !== void 0)
-        expression.argument = rewrite(expression.argument);
-      return expression;
+      if (expression2.argument !== void 0)
+        expression2.argument = rewrite(expression2.argument);
+      return expression2;
     }
-    return mapChildExpressions(expression, rewrite);
+    return mapChildExpressions(expression2, rewrite);
   };
-  mapBlockExpressions(block, rewrite);
+  return rewrite(expression);
+}
+function replacePlanReferences(block, replacements) {
+  mapBlockExpressions(block, (expression) => replaceExpressionReferences(expression, replacements));
   forEachNestedBlock(block, (nested) => replacePlanReferences(nested, replacements));
 }
 function outerProbeBlock(block, keys, resolvedNames) {
@@ -2166,30 +2202,62 @@ function blockReferencesOutside(block) {
   collectOutsideReferences(block, /* @__PURE__ */ new Set(), refs);
   return refs.length > 0;
 }
-function collectOutsideReferences(block, outer, refs) {
-  const scope = /* @__PURE__ */ new Set([...outer, block.base.alias, ...block.joins.map((join) => join.alias)]);
-  const visit = (expression) => {
-    if (expression.kind === "column") {
-      const parts = expression.reference.split(".");
+function collectExpressionOutsideReferences(expression, scope, refs) {
+  const visit = (expression2) => {
+    if (expression2.kind === "column") {
+      const parts = expression2.reference.split(".");
       const alias = parts[0];
       if (parts.length === 2 && alias !== void 0 && !scope.has(alias)) {
-        refs.push(expression.reference);
+        refs.push(expression2.reference);
       }
       return;
     }
-    if (expression.kind === "subquery" || expression.kind === "exists") {
-      collectOutsideReferences(expression.block, scope, refs);
+    if (expression2.kind === "subquery" || expression2.kind === "exists") {
+      collectOutsideReferences(expression2.block, scope, refs);
       return;
     }
-    for (const child of childExpressions(expression))
+    for (const child of childExpressions(expression2))
       visit(child);
   };
-  forEachBlockExpression(block, visit);
+  visit(expression);
+}
+function collectOutsideReferences(block, outer, refs) {
+  const scope = /* @__PURE__ */ new Set([...outer, block.base.alias, ...block.joins.map((join) => join.alias)]);
+  forEachBlockExpression(block, (expression) => collectExpressionOutsideReferences(expression, scope, refs));
   forEachNestedBlock(block, (nested) => {
     collectOutsideReferences(nested, scope, refs);
   });
 }
+function probeOuterReferences(block, additional, scope, label) {
+  const outside = [];
+  collectOutsideReferences(block, /* @__PURE__ */ new Set(), outside);
+  const references = [.../* @__PURE__ */ new Set([...outside, ...additional])];
+  if (references.length === 0) {
+    throw new TypeError(`Correlated ${label} subquery has no usable outer reference`);
+  }
+  for (const reference of references) {
+    const parts = reference.split(".");
+    const alias = parts.length === 2 ? parts[0] : void 0;
+    if (alias === void 0 || !scope.has(alias)) {
+      throw new TypeError(`Correlated ${label} references an unavailable outer source: ${reference}`);
+    }
+  }
+  return references;
+}
 function extractCorrelation(inner, outerScope, label, comparisons = false) {
+  const keys = takeCorrelationPredicates(inner, outerScope, comparisons);
+  const leftover = [];
+  collectOutsideReferences(inner, /* @__PURE__ */ new Set(), leftover);
+  if (leftover.length > 0) {
+    const supported = comparisons ? "comparison" : "equality";
+    throw new TypeError(`Correlated ${label} subqueries support only ${supported} conditions between one inner and one outer qualified column (unsupported reference: ${leftover[0] ?? ""})`);
+  }
+  if (keys.length === 0) {
+    throw new TypeError(`Correlated ${label} subquery has no usable correlation condition`);
+  }
+  return keys;
+}
+function takeCorrelationPredicates(inner, outerScope, comparisons) {
   const innerScope = /* @__PURE__ */ new Set([inner.base.alias, ...inner.joins.map((join) => join.alias)]);
   const keys = [];
   const kept = [];
@@ -2201,15 +2269,6 @@ function extractCorrelation(inner, outerScope, label, comparisons = false) {
       keys.push(key);
   }
   inner.predicates = kept;
-  const leftover = [];
-  collectOutsideReferences(inner, /* @__PURE__ */ new Set(), leftover);
-  if (leftover.length > 0) {
-    const supported = comparisons ? "comparison" : "equality";
-    throw new TypeError(`Correlated ${label} subqueries support only ${supported} conditions between one inner and one outer qualified column (unsupported reference: ${leftover[0] ?? ""})`);
-  }
-  if (keys.length === 0) {
-    throw new TypeError(`Correlated ${label} subquery has no usable correlation condition`);
-  }
   return keys;
 }
 function canExtractCorrelation(inner, outerScope, label, comparisons) {

package/dist/engine/query.js

@@ -1608,6 +1608,7 @@ function bindStatementParameters(statement, params) {
     if (clone.onConflict?.where !== void 0) {
       clone.onConflict.where = bindExpression(clone.onConflict.where, values);
     }
+    bindReturningItems(clone.returningItems, values);
     return clone;
   }
   if (clone.kind === "drop-table" || clone.kind === "create-view" || clone.kind === "drop-view" || clone.kind === "transaction") {
@@ -1643,8 +1644,15 @@ function bindStatementParameters(statement, params) {
     predicate.left = bindExpression(predicate.left, values);
     predicate.right = bindExpression(predicate.right, values);
   }
+  bindReturningItems(clone.returningItems, values);
   return clone;
 }
+function bindReturningItems(items, values) {
+  if (items === void 0)
+    return;
+  for (const item of items)
+    item.expression = bindExpression(item.expression, values);
+}
 function inferBlockSchema(plan, schemas) {
   const sources = [plan.base, ...plan.joins];
   const multipleSources = sources.filter((source) => {

package/dist/storage/indexeddb.js

@@ -3333,7 +3333,7 @@ class IndexedDbBlockStore {
         const indexKey = ftsChunkIndexKey(ftsEntry.tableId, column.columnId);
         const rawChunkIndex = await requestResult(catalog.get(indexKey));
         const chunkIndex = decodeFtsDeltaIndex(rawChunkIndex);
-        if (rawChunkIndex !== void 0 && chunkIndex === void 0 || (chunkIndex?.versions.length ?? 0) >= MAX_FTS_DELTA_CHUNKS) {
+        if (rawChunkIndex !== void 0 && chunkIndex === void 0 || postings.length > 0 && (chunkIndex?.versions.length ?? 0) >= MAX_FTS_DELTA_CHUNKS) {
           for (const version of chunkIndex?.versions ?? []) {
             catalog.delete(ftsChunkKey(ftsEntry.tableId, column.columnId, version));
           }
@@ -3356,6 +3356,14 @@ class IndexedDbBlockStore {
           });
           continue;
         }
+        if (postings.length === 0) {
+          ftsDeltaCounts.push({
+            tableId: ftsEntry.tableId,
+            columnId: column.columnId,
+            count: chunkIndex?.versions.length ?? 0
+          });
+          continue;
+        }
         catalog.put(structuredClone({
           postings,
           totalTokens: column.totalTokens

package/dist/storage/toolkit/record-core.js

@@ -1655,6 +1655,8 @@ class RecordCore {
       const active = activePostingStorageColumnIds(currentTable);
       if (!active.has(column.columnId))
         continue;
+      if (column.postings.length === 0)
+        continue;
       const key = `${changes.tableId}/${column.columnId}`;
       const deltas = this.#ftsDeltas.get(key) ?? /* @__PURE__ */ new Map();
       if (!deltas.has(version) && deltas.size >= MAX_FTS_DELTA_CHUNKS) {
@@ -4330,8 +4332,11 @@ class RecordCore {
       this.#nextAutoIncrement.set(key, next);
     for (const [key, base] of cloned.ftsBases)
       this.#ftsBases.set(key, base);
-    for (const [key, deltas] of cloned.ftsDeltas)
-      this.#ftsDeltas.set(key, new Map(deltas));
+    for (const [key, deltas] of cloned.ftsDeltas) {
+      const retained = deltas.filter(([, delta]) => delta.postings.length > 0);
+      if (retained.length > 0)
+        this.#ftsDeltas.set(key, new Map(retained));
+    }
     for (const [tableId, tokens] of cloned.uniqueKeys) {
       this.#uniqueKeys.set(tableId, new OrderedStringSet(tokens));
     }
@@ -5425,7 +5430,14 @@ function validateRecordCoreState(state, physical) {
         throw new TypeError(`Full-text delta ${key} repeats a version`);
       versions.add(version);
       whole(delta.totalTokens, `Full-text delta ${key} token count`);
-      if (validateFtsPostingChunks([delta.postings], `Full-text delta ${key}`) !== delta.totalTokens) {
+      if (!Array.isArray(delta.postings)) {
+        throw new TypeError(`Full-text delta ${key} postings must be an array`);
+      }
+      if (delta.postings.length === 0) {
+        if (delta.totalTokens !== 0) {
+          throw new TypeError(`Empty full-text delta ${key} must have a zero token total`);
+        }
+      } else if (validateFtsPostingChunks([delta.postings], `Full-text delta ${key}`) !== delta.totalTokens) {
         throw new TypeError(`Full-text delta ${key} token total does not match its postings`);
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minnowdb/core",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "A columnar SQL database for the browser: PostgreSQL-style SQL over durable IndexedDB or OPFS data, with no server or WebAssembly module.",
   "license": "MIT",
   "author": "Eric Wilhite",

package/sql-feature-matrix.json

@@ -420,8 +420,8 @@
     {
       "id": "subquery.correlated-select",
       "status": "supported",
-      "example": "SELECT r.region, (SELECT AVG(q.amount) FROM rows q WHERE q.region = r.region) AS regional FROM rows r",
-      "notes": "Correlated scalar aggregates and single-row projections decorrelate in the select list. In a grouped query, their outer references must be GROUP BY columns and the scalar cannot sit inside an outer aggregate."
+      "example": "SELECT r.region, (SELECT MAX(q.amount + r.amount) FROM rows q WHERE q.region = r.region) AS regional FROM rows r",
+      "notes": "Correlated scalar aggregates and single-row projections decorrelate in the select list. Every qualified outer column used by the inner predicates or projection becomes part of the distinct probe tuple. In a grouped query, outer references must be GROUP BY columns and the scalar cannot sit inside an outer aggregate."
     },
     {
       "id": "subquery.correlated-select-limit",