lakeql 0.2.0 → 0.5.1

package/dist/{chunk-3WUY56UD.js → chunk-TR4JNJ47.js}

@@ -1,5 +1,5 @@
-import { Lake, vectorValue, scalarVectorValue, batchFromColumns, withObjectStoreReadControls, throwIfAborted, readControlSignal, readParquetObjectBatches, readIcebergParquetDeletes, createParquetLake, parseSql, writePartitionedParquet, broadcastJoin } from './chunk-32PE6IK4.js';
-import { stableStringify } from './chunk-LWF5FKHB.js';
+import { Lake, vectorValue, scalarVectorValue, batchFromColumns, withObjectStoreReadControls, throwIfAborted, readControlSignal, readParquetObjectBatches, readIcebergParquetDeletes, createParquetLake, parseSql, writePartitionedParquet, broadcastJoin, crossJoin } from './chunk-AUL4DYT4.js';
+import { stableStringify, collectExprColumns } from './chunk-LWF5FKHB.js';
 import { LakeqlError, jsonSafeValue, matches, evaluate } from './chunk-WGHR35QU.js';
 
 // ../core/dist/in-memory.js
@@ -984,7 +984,7 @@ async function* scanPlannedIcebergRows(options) {
     throwIfAborted(signal);
     const deletes = await decodedDeletesForFile(file, options, signal);
     throwIfAborted(signal);
-    const data = await options.readDataFile(file);
+    const data = await options.readDataFile(file, deletes);
     throwIfAborted(signal);
     let rowOffset = 0;
     for await (const batch of rowBatches(data)) {
@@ -1876,15 +1876,51 @@ function projectedIds(fields, select) {
 function partitionMayMatch(expr, partition) {
   if (!expr)
     return true;
+  const value = partitionEval(expr, partition);
+  return value !== false;
+}
+function partitionEval(expr, partition) {
+  switch (expr.kind) {
+    case "literal":
+    case "column":
+      return "unknown";
+    case "compare":
+    case "in":
+    case "between":
+    case "null-check":
+    case "like":
+    case "call":
+    case "arithmetic":
+    case "case":
+      return expressionIsPartitionOnly(expr, partition) ? matchesPartition(expr, partition) : "unknown";
+    case "not": {
+      const value = partitionEval(expr.operand, partition);
+      return value === "unknown" ? "unknown" : !value;
+    }
+    case "logical": {
+      const values = expr.operands.map((operand) => partitionEval(operand, partition));
+      if (expr.op === "and") {
+        if (values.some((value) => value === false))
+          return false;
+        return values.every((value) => value === true) ? true : "unknown";
+      }
+      if (values.some((value) => value === true))
+        return true;
+      return values.every((value) => value === false) ? false : "unknown";
+    }
+  }
+}
+function expressionIsPartitionOnly(expr, partition) {
   const columns = /* @__PURE__ */ new Set();
   collectColumns(expr, columns);
-  if (columns.size === 0 || [...columns].some((column) => !(column in partition)))
-    return true;
+  return columns.size > 0 && [...columns].every((column) => column in partition);
+}
+function matchesPartition(expr, partition) {
   try {
     return matches(expr, partition);
   } catch (cause) {
     if (cause instanceof LakeqlError && cause.code === "LAKEQL_TYPE_ERROR")
-      return true;
+      return "unknown";
     throw cause;
   }
 }
@@ -1982,13 +2018,14 @@ async function* scanBatches(plan, options = {}) {
   for await (const batch of scanPlannedIcebergRows({
     plan: plan.plan,
     ...options,
-    readDataFile: async (file) => projectIcebergParquetBatches(
+    readDataFile: async (file, deletes) => projectIcebergParquetBatches(
       plan.store,
       plan.table,
       file.path,
       file.partition,
       file.snapshotId,
       plan.options,
+      deletes,
       options
     ),
     readDeleteFile: async (deleteFile) => readIcebergParquetDeletes(plan.store, deleteFile)
@@ -2003,20 +2040,34 @@ async function* scanRows(plan, options = {}) {
     }
   }
 }
-async function* projectIcebergParquetBatches(store, table, path, partition, snapshotId, planOptions, scanOptions) {
+async function* projectIcebergParquetBatches(store, table, path, partition, snapshotId, planOptions, deletes, scanOptions) {
   const readOptions = {};
   if (scanOptions.batchSize !== void 0) readOptions.batchSize = scanOptions.batchSize;
+  const retainedColumns = retainedFilterColumns(deletes, planOptions);
   for await (const batch of readParquetObjectBatches(store, path, readOptions)) {
     yield {
       rowOffset: batch.rowOffset,
       rows: batch.rows.map((row) => {
+        const sourceRow = { ...partition, ...row };
         const projectOptions = { snapshotId };
         if (planOptions.select !== void 0) projectOptions.select = planOptions.select;
-        return table.projectRow({ ...partition, ...row }, projectOptions);
+        const projected = table.projectRow(sourceRow, projectOptions);
+        for (const column of retainedColumns) {
+          if (column in sourceRow && !(column in projected)) projected[column] = sourceRow[column];
+        }
+        return projected;
       })
     };
   }
 }
+function retainedFilterColumns(deletes, planOptions) {
+  const columns = /* @__PURE__ */ new Set();
+  if (planOptions.select === void 0) collectExprColumns(planOptions.where, columns);
+  for (const deletion of deletes.equalityDeletes ?? []) {
+    for (const column of deletion.columns) columns.add(column);
+  }
+  return columns;
+}
 
 // src/sql.ts
 var textEncoder = new TextEncoder();
@@ -2066,14 +2117,32 @@ var SqlQueryResult = class {
 };
 async function executeSql(lake, sql, options) {
   const ast = sqlAst(sql, options);
+  validateSourceBindings(options);
   const bound = bindSources(ast, options);
-  const materialized = await materializeCteIfNeeded(lake.store, lake, bound);
+  const pushed = pushdownIcebergPredicates(bound, options);
+  const materializedIceberg = await materializeIcebergTablesIfNeeded(
+    lake.store,
+    pushed.ast,
+    options,
+    pushed
+  );
+  try {
+    return await executeRowsFromAst(lake, materializedIceberg.ast, options);
+  } finally {
+    await materializedIceberg.cleanup();
+  }
+}
+async function executeRowsFromAst(lake, ast, options) {
+  const materialized = await materializeCteIfNeeded(lake.store, lake, ast, options);
   try {
     const resolved = await resolveScalarSubqueries(lake, materialized.ast);
-    if (resolved.subqueryJoin !== void 0)
-      return subqueryJoinRowsFromAst(lake, resolved, options);
-    if (resolved.join !== void 0) return joinRowsFromAst(lake, resolved, options);
-    if (hasAggregation(resolved)) return aggregateRowsFromAst(lake.path(resolved.source), resolved);
+    if (sqlSubqueryJoins(resolved).length > 0)
+      return await subqueryJoinRowsFromAst(lake, resolved, options);
+    if (sqlJoins(resolved).length > 0) return await joinRowsFromAst(lake, resolved, options);
+    if (hasAggregation(resolved))
+      return await aggregateRowsFromAst(lake.path(resolved.source), resolved);
+    if (hasCorrelatedScalarSubqueries(resolved))
+      return await correlatedScalarRowsFromAst(lake, resolved);
     return await builderFromAst(lake.path(resolved.source), resolved).toArray();
   } finally {
     await materialized.cleanup();
@@ -2086,12 +2155,25 @@ function sqlAst(sql, options) {
 }
 function bindSources(ast, options) {
   const tables = options.tables ?? {};
+  const icebergTables = options.icebergTables ?? {};
   const bindSource = (source) => {
     if (source === "input" && options.path !== void 0) return options.path;
+    if (icebergTables[source] !== void 0) return source;
     return tables[source] ?? source;
   };
   return mapAstSources(ast, bindSource);
 }
+function validateSourceBindings(options) {
+  const tables = options.tables ?? {};
+  const icebergTables = options.icebergTables ?? {};
+  for (const name of Object.keys(icebergTables)) {
+    if (tables[name] === void 0) continue;
+    throw new LakeqlError(
+      "LAKEQL_VALIDATION_ERROR",
+      `SQL source "${name}" is bound as both Parquet and Iceberg`
+    );
+  }
+}
 function mapAstSources(ast, bindSource, seen = /* @__PURE__ */ new WeakSet()) {
   const out = { ...ast, source: bindSource(ast.source) };
   if (seen.has(ast)) {
@@ -2100,12 +2182,25 @@ function mapAstSources(ast, bindSource, seen = /* @__PURE__ */ new WeakSet()) {
   }
   seen.add(ast);
   if (ast.join !== void 0) out.join = { ...ast.join, source: bindSource(ast.join.source) };
-  if (ast.subqueryJoin !== void 0) {
+  if (ast.joins !== void 0) {
+    out.joins = ast.joins.map((join) => ({ ...join, source: bindSource(join.source) }));
+    const firstJoin = out.joins[0];
+    if (firstJoin !== void 0) out.join = firstJoin;
+  }
+  if (ast.subqueryJoin !== void 0 && ast.subqueryJoins === void 0) {
     out.subqueryJoin = {
       ...ast.subqueryJoin,
       source: bindSource(ast.subqueryJoin.source)
     };
   }
+  if (ast.subqueryJoins !== void 0) {
+    out.subqueryJoins = ast.subqueryJoins.map((join) => ({
+      ...join,
+      source: bindSource(join.source)
+    }));
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  }
   if (ast.cte !== void 0) {
     out.cte = { ...ast.cte, query: mapAstSources(ast.cte.query, bindSource, seen) };
   }
@@ -2117,6 +2212,69 @@ function mapAstSources(ast, bindSource, seen = /* @__PURE__ */ new WeakSet()) {
       ])
     );
   }
+  if (ast.correlatedScalarSubqueries !== void 0) {
+    out.correlatedScalarSubqueries = Object.fromEntries(
+      Object.entries(ast.correlatedScalarSubqueries).map(([id, subquery]) => [
+        id,
+        { ...subquery, source: bindSource(subquery.source) }
+      ])
+    );
+  }
+  return out;
+}
+async function mapAstSourcesAsync(ast, bindSource, seen = /* @__PURE__ */ new WeakSet()) {
+  const out = { ...ast, source: await bindSource(ast.source) };
+  if (seen.has(ast)) {
+    delete out.scalarSubqueries;
+    return out;
+  }
+  seen.add(ast);
+  if (ast.join !== void 0) {
+    out.join = { ...ast.join, source: await bindSource(ast.join.source) };
+  }
+  if (ast.joins !== void 0) {
+    out.joins = await Promise.all(
+      ast.joins.map(async (join) => ({ ...join, source: await bindSource(join.source) }))
+    );
+    const firstJoin = out.joins[0];
+    if (firstJoin !== void 0) out.join = firstJoin;
+  }
+  if (ast.subqueryJoin !== void 0 && ast.subqueryJoins === void 0) {
+    out.subqueryJoin = {
+      ...ast.subqueryJoin,
+      source: await bindSource(ast.subqueryJoin.source)
+    };
+  }
+  if (ast.subqueryJoins !== void 0) {
+    out.subqueryJoins = await Promise.all(
+      ast.subqueryJoins.map(async (join) => ({ ...join, source: await bindSource(join.source) }))
+    );
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  }
+  if (ast.cte !== void 0) {
+    out.cte = { ...ast.cte, query: await mapAstSourcesAsync(ast.cte.query, bindSource, seen) };
+  }
+  if (ast.scalarSubqueries !== void 0) {
+    out.scalarSubqueries = Object.fromEntries(
+      await Promise.all(
+        Object.entries(ast.scalarSubqueries).map(async ([id, subquery]) => [
+          id,
+          { ...subquery, query: await mapAstSourcesAsync(subquery.query, bindSource, seen) }
+        ])
+      )
+    );
+  }
+  if (ast.correlatedScalarSubqueries !== void 0) {
+    out.correlatedScalarSubqueries = Object.fromEntries(
+      await Promise.all(
+        Object.entries(ast.correlatedScalarSubqueries).map(async ([id, subquery]) => [
+          id,
+          { ...subquery, source: await bindSource(subquery.source) }
+        ])
+      )
+    );
+  }
   return out;
 }
 function builderFromAst(builder, ast) {
@@ -2138,19 +2296,10 @@ function hasAggregation(ast) {
 function hasWindowSemantics(ast) {
   return ast.windows !== void 0 && Object.keys(ast.windows).length > 0 || ast.qualify !== void 0;
 }
-async function materializeCteIfNeeded(store, lake, ast) {
+async function materializeCteIfNeeded(store, lake, ast, options) {
   if (ast.cte === void 0) return { ast, cleanup: async () => {
   } };
-  if (ast.source !== ast.cte.name) {
-    throw new LakeqlError(
-      "LAKEQL_SQL_UNSUPPORTED",
-      "CTEs are only supported as the outer FROM source"
-    );
-  }
-  if (ast.join !== void 0 || ast.subqueryJoin !== void 0) {
-    throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "CTEs inside JOINs are not supported yet");
-  }
-  const cteRows = hasAggregation(ast.cte.query) ? await aggregateRowsFromAst(lake.path(ast.cte.query.source), ast.cte.query) : await builderFromAst(lake.path(ast.cte.query.source), ast.cte.query).toArray();
+  const cteRows = await executeRowsFromAst(lake, ast.cte.query, options);
   if (cteRows.length === 0) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Empty CTE results are not supported yet");
   }
@@ -2160,13 +2309,432 @@ async function materializeCteIfNeeded(store, lake, ast) {
     maxRowsPerFile: cteRows.length
   });
   const { cte: _cte, ...rest } = ast;
+  const ctePath = `${prefix}/*.parquet`;
   return {
-    ast: { ...rest, source: `${prefix}/*.parquet` },
+    ast: mapAstSources(rest, (source) => source === ast.cte?.name ? ctePath : source),
     cleanup: async () => {
       await Promise.all(written.files.map((file) => store.delete(file.path)));
     }
   };
 }
+async function materializeIcebergTablesIfNeeded(store, ast, options, pushed = emptyIcebergPushdownResult(ast)) {
+  const icebergTables = options.icebergTables ?? {};
+  if (Object.keys(icebergTables).length === 0 && pushed.sources.size === 0) {
+    return { ast, cleanup: async () => {
+    } };
+  }
+  const materialized = /* @__PURE__ */ new Map();
+  const requiredColumns = collectSourceColumns(ast);
+  const materializeSource = async (source) => {
+    const icebergTable = pushed.sources.get(source) ?? icebergTables[source];
+    if (icebergTable === void 0) return source;
+    const cached = materialized.get(source);
+    if (cached !== void 0) return cached.path;
+    const table = await loadTable({
+      format: "iceberg",
+      store,
+      metadataPath: icebergTable.metadataPath
+    });
+    if (table.format !== "iceberg") {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `SQL source "${source}" is not Iceberg`);
+    }
+    const planOptions = icebergPlanOptions(icebergTable);
+    const sourceColumns = icebergRequiredColumns(requiredColumns, pushed.predicates, source);
+    const selectedColumns = icebergSelectableColumns(table.table, planOptions, sourceColumns);
+    if (selectedColumns !== void 0) planOptions.select = selectedColumns;
+    const predicate = pushed.predicates.get(source);
+    if (predicate !== void 0) planOptions.where = predicate;
+    const plan = planFiles2(table, planOptions);
+    const rows = [];
+    for await (const row of scanRows(plan)) rows.push(row);
+    if (rows.length === 0) {
+      throw new LakeqlError(
+        "LAKEQL_SQL_UNSUPPORTED",
+        `Iceberg SQL source "${source}" produced no rows`
+      );
+    }
+    const prefix = `__lakeql_sql_iceberg/${safeTempName(source)}/${nextSqlTempId()}`;
+    const written = await writePartitionedParquet(store, prefix, {
+      rows,
+      maxRowsPerFile: icebergTable.maxRowsPerFile ?? rows.length
+    });
+    const result = {
+      path: `${prefix}/*.parquet`,
+      files: written.files.map((file) => file.path)
+    };
+    materialized.set(source, result);
+    return result.path;
+  };
+  const mapped = await mapAstSourcesAsync(ast, materializeSource);
+  return {
+    ast: mapped,
+    cleanup: async () => {
+      await Promise.all(
+        [...materialized.values()].flatMap((table) => table.files).map((path) => store.delete(path))
+      );
+    }
+  };
+}
+function icebergRequiredColumns(requiredColumns, predicates, source) {
+  const columns = new Set(requiredColumns.get(source) ?? []);
+  const predicate = predicates.get(source);
+  if (predicate !== void 0) collectExprColumns2(predicate, columns);
+  return columns.size === 0 ? void 0 : columns;
+}
+function icebergSelectableColumns(table, planOptions, columns) {
+  if (columns === void 0) return void 0;
+  const snapshot = table.snapshot(planOptions);
+  const fields = new Set(table.schema(snapshot["schema-id"]).map((field) => field.name));
+  for (const column of columns) {
+    if (!fields.has(column)) return void 0;
+  }
+  return [...columns];
+}
+function emptyIcebergPushdownResult(ast) {
+  return { ast, predicates: /* @__PURE__ */ new Map(), sources: /* @__PURE__ */ new Map() };
+}
+function pushdownIcebergPredicates(ast, options) {
+  const icebergTables = options.icebergTables ?? {};
+  if (Object.keys(icebergTables).length === 0) return emptyIcebergPushdownResult(ast);
+  const context = {
+    icebergTables,
+    predicates: /* @__PURE__ */ new Map(),
+    sources: /* @__PURE__ */ new Map(),
+    seen: /* @__PURE__ */ new WeakSet(),
+    nextSourceId: 0
+  };
+  return {
+    ast: pushdownIcebergPredicatesFromAst(ast, context),
+    predicates: context.predicates,
+    sources: context.sources
+  };
+}
+function pushdownIcebergPredicatesFromAst(ast, context) {
+  if (context.seen.has(ast)) return ast;
+  context.seen.add(ast);
+  const out = {
+    ...ast,
+    source: icebergOccurrenceSource(ast.source, context)
+  };
+  if (ast.join !== void 0) {
+    out.join = { ...ast.join, source: icebergOccurrenceSource(ast.join.source, context) };
+  }
+  if (ast.joins !== void 0) {
+    out.joins = ast.joins.map((join) => ({
+      ...join,
+      source: icebergOccurrenceSource(join.source, context)
+    }));
+    const firstJoin = out.joins[0];
+    if (firstJoin !== void 0) out.join = firstJoin;
+  }
+  if (ast.subqueryJoin !== void 0 && ast.subqueryJoins === void 0) {
+    out.subqueryJoin = {
+      ...ast.subqueryJoin,
+      source: icebergOccurrenceSource(ast.subqueryJoin.source, context)
+    };
+  }
+  if (ast.subqueryJoins !== void 0) {
+    out.subqueryJoins = ast.subqueryJoins.map((join) => ({
+      ...join,
+      source: icebergOccurrenceSource(join.source, context)
+    }));
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  }
+  const aliases = sourceAliases(out);
+  if (ast.where !== void 0) {
+    const remaining = [];
+    for (const predicate of splitAndPredicates(ast.where)) {
+      const source = predicateSource(predicate, aliases);
+      if (source !== void 0 && context.sources.has(source)) {
+        addPushdownPredicate(
+          context.predicates,
+          source,
+          stripExprSource(predicate, aliases, source)
+        );
+      }
+      remaining.push(predicate);
+    }
+    const where = combineAndPredicates(remaining);
+    if (where === void 0) delete out.where;
+    else out.where = where;
+  }
+  const pushedSubqueryJoins = sqlSubqueryJoins(ast).map((join, index) => {
+    const outJoin = sqlSubqueryJoins(out)[index];
+    if (join.where === void 0 || outJoin === void 0) return outJoin ?? join;
+    const source = outJoin.source;
+    if (source !== void 0 && context.sources.has(source)) {
+      const subqueryAliases = /* @__PURE__ */ new Map([[source, source]]);
+      const remaining = [];
+      for (const predicate of splitAndPredicates(join.where)) {
+        const predicateTable = predicateSource(predicate, subqueryAliases);
+        if (predicateTable === source) {
+          addPushdownPredicate(
+            context.predicates,
+            source,
+            stripExprSource(predicate, subqueryAliases, source)
+          );
+        }
+        remaining.push(predicate);
+      }
+      const where = combineAndPredicates(remaining);
+      const next = { ...outJoin };
+      if (where === void 0) delete next.where;
+      else next.where = where;
+      return next;
+    }
+    return outJoin;
+  });
+  if (pushedSubqueryJoins.length > 0) {
+    out.subqueryJoins = pushedSubqueryJoins;
+    const firstSubqueryJoin = pushedSubqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  }
+  if (ast.cte !== void 0) {
+    out.cte = {
+      ...ast.cte,
+      query: pushdownIcebergPredicatesFromAst(ast.cte.query, context)
+    };
+  }
+  if (ast.scalarSubqueries !== void 0) {
+    out.scalarSubqueries = Object.fromEntries(
+      Object.entries(ast.scalarSubqueries).map(([id, subquery]) => [
+        id,
+        {
+          ...subquery,
+          query: pushdownIcebergPredicatesFromAst(subquery.query, context)
+        }
+      ])
+    );
+  }
+  if (ast.correlatedScalarSubqueries !== void 0) {
+    out.correlatedScalarSubqueries = Object.fromEntries(
+      Object.entries(ast.correlatedScalarSubqueries).map(([id, subquery]) => [
+        id,
+        { ...subquery, source: icebergOccurrenceSource(subquery.source, context) }
+      ])
+    );
+  }
+  return out;
+}
+function icebergOccurrenceSource(source, context) {
+  const table = context.icebergTables[source];
+  if (table === void 0) return source;
+  const occurrence = `__lakeql_sql_iceberg_source_${context.nextSourceId}_${safeTempName(source)}`;
+  context.nextSourceId += 1;
+  context.sources.set(occurrence, table);
+  return occurrence;
+}
+function splitAndPredicates(expr) {
+  if (expr.kind !== "logical" || expr.op !== "and") return [expr];
+  return expr.operands.flatMap(splitAndPredicates);
+}
+function combineAndPredicates(predicates) {
+  if (predicates.length === 0) return void 0;
+  if (predicates.length === 1) return predicates[0];
+  return { kind: "logical", op: "and", operands: [...predicates] };
+}
+function addPushdownPredicate(predicates, source, predicate) {
+  const existing = predicates.get(source);
+  predicates.set(
+    source,
+    existing === void 0 ? predicate : { kind: "logical", op: "and", operands: [existing, predicate] }
+  );
+}
+function predicateSource(expr, aliases) {
+  const columns = /* @__PURE__ */ new Set();
+  collectExprColumns2(expr, columns);
+  let source;
+  for (const column of columns) {
+    const columnSource = columnSourceName(column, aliases);
+    if (columnSource === void 0) return void 0;
+    if (source === void 0) source = columnSource;
+    else if (source !== columnSource) return void 0;
+  }
+  return source;
+}
+function columnSourceName(column, aliases) {
+  const dot = column.indexOf(".");
+  if (dot > 0) return aliases.get(column.slice(0, dot));
+  if (aliases.size !== 1) return void 0;
+  return aliases.values().next().value;
+}
+function stripExprSource(expr, aliases, source) {
+  switch (expr.kind) {
+    case "column":
+      return { ...expr, name: stripSourceColumn(expr.name, aliases, source) };
+    case "compare":
+      return {
+        ...expr,
+        left: stripExprSource(expr.left, aliases, source),
+        right: stripExprSource(expr.right, aliases, source)
+      };
+    case "between":
+      return {
+        ...expr,
+        target: stripExprSource(expr.target, aliases, source),
+        low: stripExprSource(expr.low, aliases, source),
+        high: stripExprSource(expr.high, aliases, source)
+      };
+    case "in":
+      return {
+        ...expr,
+        target: stripExprSource(expr.target, aliases, source),
+        values: expr.values.map((value) => stripExprSource(value, aliases, source))
+      };
+    case "logical":
+      return {
+        ...expr,
+        operands: expr.operands.map((operand) => stripExprSource(operand, aliases, source))
+      };
+    case "not":
+      return { ...expr, operand: stripExprSource(expr.operand, aliases, source) };
+    case "null-check":
+      return { ...expr, target: stripExprSource(expr.target, aliases, source) };
+    case "like":
+      return { ...expr, target: stripExprSource(expr.target, aliases, source) };
+    case "call":
+      return { ...expr, args: expr.args.map((arg) => stripExprSource(arg, aliases, source)) };
+    case "arithmetic":
+      return {
+        ...expr,
+        left: stripExprSource(expr.left, aliases, source),
+        right: stripExprSource(expr.right, aliases, source)
+      };
+    case "case":
+      return {
+        ...expr,
+        whens: expr.whens.map((branch) => ({
+          when: stripExprSource(branch.when, aliases, source),
+          value: stripExprSource(branch.value, aliases, source)
+        })),
+        ...expr.else === void 0 ? {} : { else: stripExprSource(expr.else, aliases, source) }
+      };
+    case "literal":
+      return expr;
+  }
+}
+function stripSourceColumn(column, aliases, source) {
+  const dot = column.indexOf(".");
+  if (dot <= 0) return column;
+  const qualifier = column.slice(0, dot);
+  return aliases.get(qualifier) === source ? column.slice(dot + 1) : column;
+}
+function collectSourceColumns(ast) {
+  const columns = /* @__PURE__ */ new Map();
+  collectSourceColumnsFromAst(ast, columns);
+  return columns;
+}
+function collectSourceColumnsFromAst(ast, columns, seen = /* @__PURE__ */ new WeakSet()) {
+  if (seen.has(ast)) return;
+  seen.add(ast);
+  const aliases = sourceAliases(ast);
+  const referenced = /* @__PURE__ */ new Set();
+  for (const select of ast.select ?? []) referenced.add(select);
+  for (const expr of Object.values(ast.projections ?? {})) collectExprColumns2(expr, referenced);
+  for (const aggregate of Object.values(ast.aggregates ?? {})) {
+    if (aggregate.column !== void 0) referenced.add(aggregate.column);
+    if (aggregate.expr !== void 0) collectExprColumns2(aggregate.expr, referenced);
+  }
+  for (const column of ast.groupBy ?? []) referenced.add(column);
+  for (const term of ast.orderBy ?? []) referenced.add(term.column);
+  if (ast.where !== void 0) collectExprColumns2(ast.where, referenced);
+  if (ast.having !== void 0) collectExprColumns2(ast.having, referenced);
+  if (ast.qualify !== void 0) collectExprColumns2(ast.qualify, referenced);
+  for (const window of Object.values(ast.windows ?? {})) {
+    for (const expr of window.args) collectExprColumns2(expr, referenced);
+    for (const expr of window.over.partitionBy) collectExprColumns2(expr, referenced);
+    for (const term of window.over.orderBy) collectExprColumns2(term.expr, referenced);
+    if (window.over.frame?.start.offset !== void 0)
+      collectExprColumns2(window.over.frame.start.offset, referenced);
+    if (window.over.frame?.end.offset !== void 0)
+      collectExprColumns2(window.over.frame.end.offset, referenced);
+    if (window.filter !== void 0) collectExprColumns2(window.filter, referenced);
+  }
+  for (const join of sqlJoins(ast)) {
+    for (const column of join.leftKey) referenced.add(column);
+    for (const column of join.rightKey) referenced.add(column);
+    if (join.predicate !== void 0) collectExprColumns2(join.predicate, referenced);
+  }
+  for (const subqueryJoin of sqlSubqueryJoins(ast)) {
+    for (const column of subqueryJoin.leftKey) referenced.add(column);
+    for (const column of subqueryJoin.rightKey) referenced.add(column);
+    for (const column of subqueryJoin.groupBy ?? []) referenced.add(column);
+    for (const aggregate of Object.values(subqueryJoin.aggregates ?? {})) {
+      if (aggregate.column !== void 0) referenced.add(aggregate.column);
+      if (aggregate.expr !== void 0) collectExprColumns2(aggregate.expr, referenced);
+    }
+    if (subqueryJoin.where !== void 0) collectExprColumns2(subqueryJoin.where, referenced);
+    if (subqueryJoin.having !== void 0) collectExprColumns2(subqueryJoin.having, referenced);
+    if (subqueryJoin.predicate !== void 0)
+      collectExprColumns2(subqueryJoin.predicate, referenced);
+    for (const term of subqueryJoin.orderBy ?? []) referenced.add(term.column);
+  }
+  for (const subquery of Object.values(ast.correlatedScalarSubqueries ?? {})) {
+    for (const column of subquery.leftKey) referenced.add(column);
+    for (const column of subquery.rightKey) addColumn(columns, subquery.source, column);
+    for (const column of subquery.groupBy) addColumn(columns, subquery.source, column);
+    for (const aggregate of Object.values(subquery.aggregates)) {
+      if (aggregate.column !== void 0) addColumn(columns, subquery.source, aggregate.column);
+      if (aggregate.expr !== void 0)
+        collectExprColumnsForSource(aggregate.expr, columns, subquery.source);
+    }
+    if (subquery.where !== void 0)
+      collectExprColumnsForSource(subquery.where, columns, subquery.source);
+    if (subquery.having !== void 0)
+      collectExprColumnsForSource(subquery.having, columns, subquery.source);
+  }
+  for (const column of referenced) addSourceColumn(columns, aliases, column);
+  if (ast.cte !== void 0) collectSourceColumnsFromAst(ast.cte.query, columns, seen);
+  for (const subquery of Object.values(ast.scalarSubqueries ?? {})) {
+    collectSourceColumnsFromAst(subquery.query, columns, seen);
+  }
+}
+function sourceAliases(ast) {
+  const aliases = /* @__PURE__ */ new Map([[ast.source, ast.source]]);
+  const joins = sqlJoins(ast);
+  const firstJoin = joins[0];
+  if (firstJoin !== void 0) aliases.set(firstJoin.leftAlias, ast.source);
+  for (const join of joins) {
+    aliases.set(join.source, join.source);
+    aliases.set(join.alias, join.source);
+  }
+  return aliases;
+}
+function addSourceColumn(columns, aliases, column) {
+  if (column === "*" || column.endsWith(".*")) return;
+  const dot = column.indexOf(".");
+  if (dot > 0) {
+    const source2 = aliases.get(column.slice(0, dot));
+    if (source2 === void 0) return;
+    addColumn(columns, source2, column.slice(dot + 1));
+    return;
+  }
+  if (aliases.size !== 1) return;
+  const source = aliases.values().next().value;
+  if (source !== void 0) addColumn(columns, source, column);
+}
+function collectExprColumnsForSource(expr, columns, source) {
+  const referenced = /* @__PURE__ */ new Set();
+  collectExprColumns2(expr, referenced);
+  for (const column of referenced) addColumn(columns, source, column);
+}
+function addColumn(columns, source, column) {
+  let sourceColumns = columns.get(source);
+  if (sourceColumns === void 0) {
+    sourceColumns = /* @__PURE__ */ new Set();
+    columns.set(source, sourceColumns);
+  }
+  sourceColumns.add(column);
+}
+function icebergPlanOptions(table) {
+  const options = {};
+  if (table.snapshotId !== void 0) options.snapshotId = table.snapshotId;
+  if (table.asOfTimestampMs !== void 0) options.asOfTimestampMs = table.asOfTimestampMs;
+  if (table.ref !== void 0) options.ref = table.ref;
+  if (table.readMode !== void 0) options.readMode = table.readMode;
+  return options;
+}
 function nextSqlTempId() {
   const randomUuid = globalThis.crypto?.randomUUID?.();
   if (randomUuid !== void 0) return randomUuid;
@@ -2181,6 +2749,10 @@ async function resolveScalarSubqueries(lake, ast) {
   const values = /* @__PURE__ */ new Map();
   for (const [id, subquery] of Object.entries(ast.scalarSubqueries)) {
     const rows = hasAggregation(subquery.query) ? await aggregateRowsFromAst(lake.path(subquery.query.source), subquery.query) : await builderFromAst(lake.path(subquery.query.source), subquery.query).toArray();
+    if (subquery.mode === "exists") {
+      values.set(id, rows.length > 0);
+      continue;
+    }
     if (rows.length > 1) {
       throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Scalar subquery returned more than one row");
     }
@@ -2197,6 +2769,30 @@ async function resolveScalarSubqueries(lake, ast) {
       ])
     );
   }
+  if (ast.joins !== void 0) {
+    out.joins = ast.joins.map(
+      (join) => join.predicate === void 0 ? join : { ...join, predicate: replaceScalarSubqueryExpr(join.predicate, values) }
+    );
+    const firstJoin = out.joins[0];
+    if (firstJoin !== void 0) out.join = firstJoin;
+  } else if (ast.join?.predicate !== void 0) {
+    out.join = { ...ast.join, predicate: replaceScalarSubqueryExpr(ast.join.predicate, values) };
+  }
+  if (ast.subqueryJoins !== void 0) {
+    out.subqueryJoins = ast.subqueryJoins.map((join) => ({
+      ...join,
+      ...join.where === void 0 ? {} : { where: replaceScalarSubqueryExpr(join.where, values) },
+      ...join.predicate === void 0 ? {} : { predicate: replaceScalarSubqueryExpr(join.predicate, values) }
+    }));
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  } else if (ast.subqueryJoin?.where !== void 0 || ast.subqueryJoin?.predicate !== void 0) {
+    out.subqueryJoin = {
+      ...ast.subqueryJoin,
+      ...ast.subqueryJoin.where === void 0 ? {} : { where: replaceScalarSubqueryExpr(ast.subqueryJoin.where, values) },
+      ...ast.subqueryJoin.predicate === void 0 ? {} : { predicate: replaceScalarSubqueryExpr(ast.subqueryJoin.predicate, values) }
+    };
+  }
   delete out.scalarSubqueries;
   return out;
 }
@@ -2262,221 +2858,402 @@ function replaceScalarSubqueryExpr(expr, values) {
   }
 }
 async function joinRowsFromAst(lake, ast, options) {
-  if (ast.join === void 0) throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
-  const join = ast.join;
+  const joins = sqlJoins(ast);
+  const firstJoin = joins[0];
+  if (firstJoin === void 0) throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over JOIN is not supported");
   }
   if (hasAggregation(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Aggregate SQL over JOIN is not supported yet");
   }
-  const leftAlias = ast.join.leftAlias;
-  const plan = planJoinSides(ast, leftAlias, join.alias);
-  let leftBuilder = lake.path(ast.source);
-  let rightBuilder = lake.path(join.source);
-  if (plan.leftWhere !== void 0) leftBuilder = leftBuilder.where(plan.leftWhere);
-  if (plan.rightWhere !== void 0) rightBuilder = rightBuilder.where(plan.rightWhere);
-  if (plan.leftColumns !== void 0) leftBuilder = leftBuilder.select(plan.leftColumns);
-  if (plan.rightColumns !== void 0) rightBuilder = rightBuilder.select(plan.rightColumns);
-  const leftRows = (await leftBuilder.toArray()).map((row) => qualifyRow(row, leftAlias));
-  const rightRows = (await rightBuilder.toArray()).map((row) => qualifyOnlyRow(row, join.alias));
-  let rows = await broadcastJoin(leftRows, rightRows, {
-    leftKey: join.leftKey,
-    rightKey: join.rightKey,
-    type: join.type,
-    rightPrefix: `${join.alias}.`,
-    maxRightRows: options.joinMaxRightRows ?? 1e5
-  });
-  if (join.type === "left") {
-    rows = fillLeftJoinNulls(rows, join.alias, leftJoinRightColumns(ast, join.alias, rightRows));
+  const pushedFilters = pushdownJoinFilters(ast, joins);
+  const readColumns = joinReadColumns(ast, joins);
+  let leftRows = await rowsForJoinAlias(lake.path(ast.source), readColumns, firstJoin.leftAlias);
+  const leftFilter = pushedFilters.filters.get(firstJoin.leftAlias);
+  if (leftFilter !== void 0) leftRows = leftRows.filter((row) => matches(leftFilter, row));
+  let rows = leftRows.map((row) => qualifyRow(row, firstJoin.leftAlias));
+  for (const join of joins) {
+    let rawRightRows = await rowsForJoinAlias(lake.path(join.source), readColumns, join.alias);
+    const rightFilter = pushedFilters.filters.get(join.alias);
+    if (rightFilter !== void 0)
+      rawRightRows = rawRightRows.filter((row) => matches(rightFilter, row));
+    const rightRows = rawRightRows.map((row) => qualifyOnlyRow(row, join.alias));
+    if (join.predicate !== void 0) {
+      rows = predicateJoinRows(rows, rightRows, ast, join, options);
+    } else if (join.type === "right") {
+      const leftRows2 = rows;
+      rows = await broadcastJoin(rightRows, leftRows2, {
+        leftKey: join.rightKey,
+        rightKey: join.leftKey,
+        type: "left",
+        rightPrefix: `${join.leftAlias}.`,
+        maxRightRows: options.joinMaxRightRows ?? 1e5
+      });
+      rows = fillRightJoinNulls(rows, ast, join.alias, leftRows2);
+    } else if (join.type === "full") {
+      const leftRows2 = rows;
+      const leftJoined = await broadcastJoin(leftRows2, rightRows, {
+        leftKey: join.leftKey,
+        rightKey: join.rightKey,
+        type: "left",
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: options.joinMaxRightRows ?? 1e5
+      });
+      const unmatchedRight = await broadcastJoin(rightRows, leftRows2, {
+        leftKey: join.rightKey,
+        rightKey: join.leftKey,
+        type: "anti",
+        rightPrefix: `${join.leftAlias}.`,
+        maxRightRows: options.joinMaxRightRows ?? 1e5
+      });
+      rows = [
+        ...fillLeftJoinNulls(
+          leftJoined,
+          join.alias,
+          leftJoinRightColumns(ast, join.alias, rightRows)
+        ),
+        ...fillRightJoinNulls(unmatchedRight, ast, join.alias, leftRows2)
+      ];
+    } else {
+      rows = join.type === "cross" ? await crossJoin(rows, rightRows, {
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: options.joinMaxRightRows ?? 1e5,
+        maxOutputRows: options.joinMaxOutputRows ?? 1e5
+      }) : await broadcastJoin(rows, rightRows, {
+        leftKey: join.leftKey,
+        rightKey: join.rightKey,
+        type: join.type,
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: options.joinMaxRightRows ?? 1e5
+      });
+    }
+    if (join.type === "left") {
+      rows = fillLeftJoinNulls(rows, join.alias, leftJoinRightColumns(ast, join.alias, rightRows));
+    }
   }
-  if (plan.residualWhere !== void 0)
-    rows = rows.filter((row) => matches(plan.residualWhere, row));
-  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
-  rows = projectRows(rows, ast);
+  if (pushedFilters.where !== void 0)
+    rows = rows.filter((row) => matches(pushedFilters.where, row));
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy, ast);
+  rows = await projectRows(lake, rows, ast);
   if (ast.distinct === true) rows = distinctRows(rows);
   return offsetLimitRows(rows, ast);
 }
-function planJoinSides(ast, leftAlias, rightAlias) {
-  if (ast.join === void 0) throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
-  const leftPredicates = [];
-  const rightPredicates = [];
-  const residualPredicates = [];
-  for (const predicate of splitAndPredicate(ast.where)) {
-    const columns = exprColumns(predicate);
-    if (columns.length > 0 && columns.every((column) => isQualifiedBy(column, leftAlias))) {
-      leftPredicates.push(stripQualifiedExpr(predicate, leftAlias));
-    } else if (ast.join.type === "inner" && columns.length > 0 && columns.every((column) => isQualifiedBy(column, rightAlias))) {
-      rightPredicates.push(stripQualifiedExpr(predicate, rightAlias));
-    } else {
-      residualPredicates.push(predicate);
+function predicateJoinRows(leftRows, rightRows, ast, join, options) {
+  if (join.predicate === void 0) return leftRows;
+  const maxRightRows = options.joinMaxRightRows ?? 1e5;
+  if (rightRows.length > maxRightRows) {
+    throw new LakeqlError(
+      "LAKEQL_BUDGET_EXCEEDED",
+      `Predicate join exceeded maxRightRows (${rightRows.length} > ${maxRightRows})`,
+      { metric: "maxRightRows", limit: maxRightRows, actual: rightRows.length }
+    );
+  }
+  const maxOutputRows = options.joinMaxOutputRows ?? 1e5;
+  const matchedLeft = /* @__PURE__ */ new Set();
+  const matchedRight = /* @__PURE__ */ new Set();
+  const out = [];
+  let candidateRows = 0;
+  for (const [leftIndex, leftRow] of leftRows.entries()) {
+    for (const [rightIndex, rightRow] of rightRows.entries()) {
+      candidateRows += 1;
+      if (candidateRows > maxOutputRows) {
+        throw new LakeqlError(
+          "LAKEQL_BUDGET_EXCEEDED",
+          `Predicate join exceeded maxOutputRows (${candidateRows} > ${maxOutputRows})`,
+          { metric: "maxOutputRows", limit: maxOutputRows, actual: candidateRows }
+        );
+      }
+      const joined = mergePredicateJoinRows(leftRow, rightRow, `${join.alias}.`);
+      if (!matches(join.predicate, joined)) continue;
+      matchedLeft.add(leftIndex);
+      matchedRight.add(rightIndex);
+      out.push(joined);
+    }
+  }
+  if (join.type === "left" || join.type === "full") {
+    const unmatchedLeft = leftRows.filter((_row, index) => !matchedLeft.has(index));
+    out.push(
+      ...fillLeftJoinNulls(
+        unmatchedLeft,
+        join.alias,
+        leftJoinRightColumns(ast, join.alias, rightRows)
+      )
+    );
+  }
+  if (join.type === "right" || join.type === "full") {
+    const unmatchedRight = rightRows.filter((_row, index) => !matchedRight.has(index));
+    out.push(...fillRightJoinNulls(unmatchedRight, ast, join.alias, leftRows));
+  }
+  return out;
+}
+function mergePredicateJoinRows(left, right, rightPrefix) {
+  const out = { ...left };
+  for (const [key, value] of Object.entries(right)) {
+    const outKey = key in out ? `${rightPrefix}${key}` : key;
+    out[outKey] = value;
+  }
+  return out;
+}
+function sqlJoins(ast) {
+  if (ast.joins !== void 0) return ast.joins;
+  return ast.join === void 0 ? [] : [ast.join];
+}
+function sqlSubqueryJoins(ast) {
+  if (ast.subqueryJoins !== void 0) return ast.subqueryJoins;
+  return ast.subqueryJoin === void 0 ? [] : [ast.subqueryJoin];
+}
+function pushdownJoinFilters(ast, joins) {
+  const filters = /* @__PURE__ */ new Map();
+  if (ast.where === void 0) {
+    return { filters, ...ast.where === void 0 ? {} : { where: ast.where } };
+  }
+  const removableAliases = nonNullableJoinAliases(joins);
+  const aliases = /* @__PURE__ */ new Set();
+  const firstJoin = joins[0];
+  if (firstJoin !== void 0) aliases.add(firstJoin.leftAlias);
+  for (const join of joins) aliases.add(join.alias);
+  const remaining = [];
+  for (const predicate of splitAndPredicates(ast.where)) {
+    const alias = predicateJoinAlias(predicate, aliases);
+    if (alias === void 0) {
+      remaining.push(predicate);
+      continue;
     }
+    addJoinFilter(filters, alias, stripExprJoinAlias(predicate, alias));
+    if (!removableAliases.has(alias)) remaining.push(predicate);
+  }
+  const where = combineAndPredicates(remaining);
+  return { filters, ...where === void 0 ? {} : { where } };
+}
+function nonNullableJoinAliases(joins) {
+  const aliases = /* @__PURE__ */ new Set();
+  const firstJoin = joins[0];
+  if (firstJoin === void 0) return aliases;
+  aliases.add(firstJoin.leftAlias);
+  for (const join of joins) {
+    switch (join.type) {
+      case "inner":
+      case "cross":
+      case "left":
+        if (join.type !== "left") aliases.add(join.alias);
+        break;
+      case "right":
+        aliases.clear();
+        aliases.add(join.alias);
+        break;
+      case "full":
+        aliases.clear();
+        break;
+    }
+  }
+  return aliases;
+}
+async function rowsForJoinAlias(builder, readColumns, alias) {
+  const columns = readColumns?.get(alias);
+  if (columns === void 0 || columns.size === 0) return await builder.toArray();
+  return await builder.select([...columns]).toArray();
+}
+function joinReadColumns(ast, joins) {
+  const aliases = /* @__PURE__ */ new Set();
+  const firstJoin = joins[0];
+  if (firstJoin !== void 0) aliases.add(firstJoin.leftAlias);
+  for (const join of joins) aliases.add(join.alias);
+  const columns = /* @__PURE__ */ new Map();
+  for (const select of ast.select ?? []) {
+    const { column } = selectColumn(select);
+    if (!addJoinReadColumn(columns, aliases, column)) return void 0;
+  }
+  for (const expr of Object.values(ast.projections ?? {})) {
+    if (!addJoinReadExpr(columns, aliases, expr)) return void 0;
+  }
+  if (ast.where !== void 0 && !addJoinReadExpr(columns, aliases, ast.where)) return void 0;
+  for (const term of ast.orderBy ?? []) {
+    if (!addJoinOrderByReadColumns(columns, aliases, term.column, ast)) return void 0;
   }
-  const plan = {};
-  const leftWhere = combineAndPredicate(leftPredicates);
-  const rightWhere = combineAndPredicate(rightPredicates);
-  const residualWhere = combineAndPredicate(residualPredicates);
-  if (leftWhere !== void 0) plan.leftWhere = leftWhere;
-  if (rightWhere !== void 0) plan.rightWhere = rightWhere;
-  if (residualWhere !== void 0) plan.residualWhere = residualWhere;
-  if (canProjectJoinSides(ast, leftAlias, rightAlias)) {
-    plan.leftColumns = joinSideColumns(ast, leftAlias, ast.join.leftKey);
-    plan.rightColumns = joinSideColumns(ast, rightAlias, ast.join.rightKey);
+  for (const join of joins) {
+    for (const column of join.leftKey) {
+      if (!addJoinReadColumn(columns, aliases, column)) return void 0;
+    }
+    for (const column of join.rightKey) {
+      if (!addJoinReadColumn(columns, aliases, column)) return void 0;
+    }
+    if (join.predicate !== void 0 && !addJoinReadExpr(columns, aliases, join.predicate)) {
+      return void 0;
+    }
   }
-  return plan;
+  return columns;
 }
-function splitAndPredicate(expr) {
-  if (expr === void 0) return [];
-  if (expr.kind === "logical" && expr.op === "and") return expr.operands.flatMap(splitAndPredicate);
-  return [expr];
+function addJoinOrderByReadColumns(columns, aliases, column, ast) {
+  const projection = ast.projections?.[column];
+  if (projection !== void 0) return addJoinReadExpr(columns, aliases, projection);
+  const select = ast.select?.find((value) => selectColumn(value).alias === column);
+  if (select !== void 0) return addJoinReadColumn(columns, aliases, selectColumn(select).column);
+  return addJoinReadColumn(columns, aliases, column);
 }
-function combineAndPredicate(predicates) {
-  if (predicates.length === 0) return void 0;
-  if (predicates.length === 1) return predicates[0];
-  return { kind: "logical", op: "and", operands: predicates };
+function addJoinReadExpr(columns, aliases, expr) {
+  const referenced = /* @__PURE__ */ new Set();
+  collectExprColumns2(expr, referenced);
+  for (const column of referenced) {
+    if (!addJoinReadColumn(columns, aliases, column)) return false;
+  }
+  return true;
 }
-function isQualifiedBy(column, alias) {
-  return column.startsWith(`${alias}.`) && column.length > alias.length + 1;
+function addJoinReadColumn(columns, aliases, column) {
+  if (column === "*" || column.endsWith(".*")) return false;
+  const dot = column.indexOf(".");
+  if (dot <= 0) return false;
+  const alias = column.slice(0, dot);
+  if (!aliases.has(alias)) return false;
+  let aliasColumns = columns.get(alias);
+  if (aliasColumns === void 0) {
+    aliasColumns = /* @__PURE__ */ new Set();
+    columns.set(alias, aliasColumns);
+  }
+  aliasColumns.add(column.slice(dot + 1));
+  return true;
 }
-function stripQualifiedColumn(column, alias) {
-  return isQualifiedBy(column, alias) ? column.slice(alias.length + 1) : column;
+function predicateJoinAlias(expr, aliases) {
+  const columns = /* @__PURE__ */ new Set();
+  collectExprColumns2(expr, columns);
+  let alias;
+  for (const column of columns) {
+    const dot = column.indexOf(".");
+    if (dot <= 0) return void 0;
+    const columnAlias = column.slice(0, dot);
+    if (!aliases.has(columnAlias)) return void 0;
+    if (alias === void 0) alias = columnAlias;
+    else if (alias !== columnAlias) return void 0;
+  }
+  return alias;
+}
+function addJoinFilter(filters, alias, predicate) {
+  const existing = filters.get(alias);
+  filters.set(
+    alias,
+    existing === void 0 ? predicate : { kind: "logical", op: "and", operands: [existing, predicate] }
+  );
 }
-function stripQualifiedExpr(expr, alias) {
+function stripExprJoinAlias(expr, alias) {
   switch (expr.kind) {
     case "column":
       return { ...expr, name: stripQualifiedColumn(expr.name, alias) };
     case "compare":
       return {
         ...expr,
-        left: stripQualifiedExpr(expr.left, alias),
-        right: stripQualifiedExpr(expr.right, alias)
+        left: stripExprJoinAlias(expr.left, alias),
+        right: stripExprJoinAlias(expr.right, alias)
       };
     case "between":
       return {
         ...expr,
-        target: stripQualifiedExpr(expr.target, alias),
-        low: stripQualifiedExpr(expr.low, alias),
-        high: stripQualifiedExpr(expr.high, alias)
+        target: stripExprJoinAlias(expr.target, alias),
+        low: stripExprJoinAlias(expr.low, alias),
+        high: stripExprJoinAlias(expr.high, alias)
       };
     case "in":
       return {
         ...expr,
-        target: stripQualifiedExpr(expr.target, alias),
-        values: expr.values.map((value) => stripQualifiedExpr(value, alias))
+        target: stripExprJoinAlias(expr.target, alias),
+        values: expr.values.map((value) => stripExprJoinAlias(value, alias))
       };
     case "logical":
       return {
         ...expr,
-        operands: expr.operands.map((operand) => stripQualifiedExpr(operand, alias))
+        operands: expr.operands.map((operand) => stripExprJoinAlias(operand, alias))
       };
     case "not":
-      return { ...expr, operand: stripQualifiedExpr(expr.operand, alias) };
+      return { ...expr, operand: stripExprJoinAlias(expr.operand, alias) };
     case "null-check":
-      return { ...expr, target: stripQualifiedExpr(expr.target, alias) };
+      return { ...expr, target: stripExprJoinAlias(expr.target, alias) };
     case "like":
-      return { ...expr, target: stripQualifiedExpr(expr.target, alias) };
+      return { ...expr, target: stripExprJoinAlias(expr.target, alias) };
     case "call":
-      return { ...expr, args: expr.args.map((arg) => stripQualifiedExpr(arg, alias)) };
+      return { ...expr, args: expr.args.map((arg) => stripExprJoinAlias(arg, alias)) };
     case "arithmetic":
       return {
         ...expr,
-        left: stripQualifiedExpr(expr.left, alias),
-        right: stripQualifiedExpr(expr.right, alias)
+        left: stripExprJoinAlias(expr.left, alias),
+        right: stripExprJoinAlias(expr.right, alias)
       };
     case "case":
       return {
         ...expr,
         whens: expr.whens.map((branch) => ({
-          when: stripQualifiedExpr(branch.when, alias),
-          value: stripQualifiedExpr(branch.value, alias)
+          when: stripExprJoinAlias(branch.when, alias),
+          value: stripExprJoinAlias(branch.value, alias)
         })),
-        ...expr.else === void 0 ? {} : { else: stripQualifiedExpr(expr.else, alias) }
+        ...expr.else === void 0 ? {} : { else: stripExprJoinAlias(expr.else, alias) }
       };
     case "literal":
       return expr;
   }
 }
-function exprColumns(expr) {
-  const columns = /* @__PURE__ */ new Set();
-  collectExprColumns(expr, columns);
-  return [...columns];
+function isQualifiedBy(column, alias) {
+  return column.startsWith(`${alias}.`) && column.length > alias.length + 1;
+}
+function stripQualifiedColumn(column, alias) {
+  return isQualifiedBy(column, alias) ? column.slice(alias.length + 1) : column;
 }
-function collectExprColumns(expr, columns) {
+function collectExprColumns2(expr, columns) {
   switch (expr.kind) {
     case "column":
       columns.add(expr.name);
       return;
     case "compare":
-      collectExprColumns(expr.left, columns);
-      collectExprColumns(expr.right, columns);
+      collectExprColumns2(expr.left, columns);
+      collectExprColumns2(expr.right, columns);
       return;
     case "between":
-      collectExprColumns(expr.target, columns);
-      collectExprColumns(expr.low, columns);
-      collectExprColumns(expr.high, columns);
+      collectExprColumns2(expr.target, columns);
+      collectExprColumns2(expr.low, columns);
+      collectExprColumns2(expr.high, columns);
       return;
     case "in":
-      collectExprColumns(expr.target, columns);
-      for (const value of expr.values) collectExprColumns(value, columns);
+      collectExprColumns2(expr.target, columns);
+      for (const value of expr.values) collectExprColumns2(value, columns);
       return;
     case "logical":
-      for (const operand of expr.operands) collectExprColumns(operand, columns);
+      for (const operand of expr.operands) collectExprColumns2(operand, columns);
       return;
     case "not":
-      collectExprColumns(expr.operand, columns);
+      collectExprColumns2(expr.operand, columns);
       return;
     case "null-check":
-      collectExprColumns(expr.target, columns);
+      collectExprColumns2(expr.target, columns);
       return;
     case "like":
-      collectExprColumns(expr.target, columns);
+      collectExprColumns2(expr.target, columns);
       return;
     case "call":
-      for (const arg of expr.args) collectExprColumns(arg, columns);
+      for (const arg of expr.args) collectExprColumns2(arg, columns);
       return;
     case "arithmetic":
-      collectExprColumns(expr.left, columns);
-      collectExprColumns(expr.right, columns);
+      collectExprColumns2(expr.left, columns);
+      collectExprColumns2(expr.right, columns);
       return;
     case "case":
       for (const branch of expr.whens) {
-        collectExprColumns(branch.when, columns);
-        collectExprColumns(branch.value, columns);
+        collectExprColumns2(branch.when, columns);
+        collectExprColumns2(branch.value, columns);
       }
-      if (expr.else !== void 0) collectExprColumns(expr.else, columns);
+      if (expr.else !== void 0) collectExprColumns2(expr.else, columns);
       return;
     case "literal":
       return;
   }
 }
-function canProjectJoinSides(ast, leftAlias, rightAlias) {
-  if (ast.select?.includes("*") ?? false) return false;
-  return referencedJoinColumns(ast).every(
-    (column) => isQualifiedBy(column, leftAlias) || isQualifiedBy(column, rightAlias)
-  );
-}
 function referencedJoinColumns(ast) {
   const columns = /* @__PURE__ */ new Set();
   for (const select of ast.select ?? []) {
     const { column } = selectColumn(select);
     columns.add(column);
   }
-  for (const expr of Object.values(ast.projections ?? {})) collectExprColumns(expr, columns);
-  if (ast.where !== void 0) collectExprColumns(ast.where, columns);
+  for (const expr of Object.values(ast.projections ?? {})) collectExprColumns2(expr, columns);
+  if (ast.where !== void 0) collectExprColumns2(ast.where, columns);
   for (const term of ast.orderBy ?? []) columns.add(term.column);
   return [...columns].filter((column) => column !== "*");
 }
-function joinSideColumns(ast, alias, joinKey2) {
-  const columns = new Set(
-    (Array.isArray(joinKey2) ? joinKey2 : [joinKey2]).map(
-      (column) => stripQualifiedColumn(column, alias)
-    )
-  );
-  for (const column of referencedJoinColumns(ast)) {
-    if (isQualifiedBy(column, alias)) columns.add(stripQualifiedColumn(column, alias));
-  }
-  return [...columns];
-}
 function leftJoinRightColumns(ast, rightAlias, rightRows) {
   const columns = /* @__PURE__ */ new Set();
   if (ast.select?.includes("*") ?? false) {
@@ -2505,33 +3282,213 @@ function fillLeftJoinNulls(rows, rightAlias, rightColumns) {
     return out ?? row;
   });
 }
+function fillRightJoinNulls(rows, ast, preservedAlias, leftRows) {
+  const columns = rightJoinLeftColumns(ast, preservedAlias, leftRows);
+  if (columns.length === 0) return rows;
+  return rows.map((row) => {
+    let out;
+    for (const column of columns) {
+      if (column in row) continue;
+      out ??= { ...row };
+      out[column] = null;
+    }
+    return out ?? row;
+  });
+}
+function rightJoinLeftColumns(ast, preservedAlias, leftRows) {
+  const columns = /* @__PURE__ */ new Set();
+  if (ast.select?.includes("*") ?? false) {
+    for (const row of leftRows) {
+      for (const column of Object.keys(row)) {
+        if (!isQualifiedBy(column, preservedAlias)) columns.add(column);
+      }
+    }
+  }
+  for (const column of referencedJoinColumns(ast)) {
+    if (!isQualifiedBy(column, preservedAlias)) columns.add(column);
+  }
+  return [...columns];
+}
 async function subqueryJoinRowsFromAst(lake, ast, options) {
-  if (ast.subqueryJoin === void 0) {
+  const joins = sqlSubqueryJoins(ast);
+  if (joins.length === 0) {
     throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL IN subquery");
   }
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over IN subquery is not supported");
   }
   if (hasAggregation(ast)) {
-    throw new LakeqlError(
-      "LAKEQL_SQL_UNSUPPORTED",
-      "Aggregate SQL over IN subquery is not supported yet"
-    );
+    return await aggregateSubqueryJoinRowsFromAst(lake, ast, joins, options);
+  }
+  let rows = await lake.path(ast.source).toArray();
+  for (const join of joins) {
+    rows = await applySubqueryJoinRows(lake, rows, join, options);
+  }
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy, ast);
+  rows = await projectRows(lake, rows, ast);
+  if (ast.distinct === true) rows = distinctRows(rows);
+  return offsetLimitRows(rows, ast);
+}
+async function aggregateSubqueryJoinRowsFromAst(lake, ast, joins, options) {
+  let rows = await lake.path(ast.source).toArray();
+  for (const join of joins) rows = await applySubqueryJoinRows(lake, rows, join, options);
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
+  const aggregateAst = subqueryJoinAggregateAst(ast);
+  if (rows.length === 0) {
+    if ((ast.groupBy?.length ?? 0) > 0) return [];
+    return emptyAggregateRowsFromAst(aggregateAst);
+  }
+  const prefix = `__lakeql_sql_subquery/${safeTempName(ast.source)}/${nextSqlTempId()}`;
+  const written = await writePartitionedParquet(lake.store, prefix, {
+    rows,
+    maxRowsPerFile: rows.length
+  });
+  try {
+    return await aggregateRowsFromAst(lake.path(`${prefix}/*.parquet`), {
+      ...aggregateAst,
+      source: `${prefix}/*.parquet`
+    });
+  } finally {
+    await Promise.all(written.files.map((file) => lake.store.delete(file.path)));
   }
-  const join = ast.subqueryJoin;
-  let rightRows = await lake.path(join.source).toArray();
-  if (join.where !== void 0) rightRows = rightRows.filter((row) => matches(join.where, row));
-  let rows = await broadcastJoin(await lake.path(ast.source).toArray(), rightRows, {
+}
+function subqueryJoinAggregateAst(ast) {
+  const {
+    subqueryJoin: _subqueryJoin,
+    subqueryJoins: _subqueryJoins,
+    where: _where,
+    ...rest
+  } = ast;
+  return rest;
+}
+function emptyAggregateRowsFromAst(ast) {
+  validateAggregateProjection(ast);
+  const hiddenCountAlias = hiddenAggregateAlias(ast);
+  const aggregateSpec = ast.aggregates === void 0 || Object.keys(ast.aggregates).length === 0 ? { [hiddenCountAlias]: { op: "count" } } : ast.aggregates;
+  let rows = [emptyAggregateRow(aggregateSpec)];
+  if (ast.having !== void 0) rows = rows.filter((row) => matches(ast.having, row));
+  rows = projectAggregateRows(rows, ast, hiddenCountAlias);
+  if (ast.distinct === true) rows = distinctRows(rows);
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy, ast);
+  const offset = ast.offset ?? 0;
+  if (ast.limit !== void 0) rows = rows.slice(offset, offset + ast.limit);
+  else if (offset > 0) rows = rows.slice(offset);
+  return rows;
+}
+function emptyAggregateRow(spec) {
+  return Object.fromEntries(
+    Object.entries(spec).map(([alias, aggregate]) => [alias, emptyAggregateValue(aggregate.op)])
+  );
+}
+function emptyAggregateValue(op) {
+  switch (op) {
+    case "count":
+    case "count_distinct":
+    case "approx_count_distinct":
+      return 0;
+    case "sum":
+    case "avg":
+    case "var_samp":
+    case "var_pop":
+    case "stddev_samp":
+    case "stddev_pop":
+    case "median":
+    case "quantile":
+    case "min":
+    case "max":
+    case "mode":
+    case "first":
+    case "last":
+    case "any":
+      return null;
+  }
+}
+async function applySubqueryJoinRows(lake, leftRows, join, options) {
+  let rightRows = await subqueryJoinRightRows(lake, join);
+  rightRows = offsetLimitSubqueryRows(rightRows, join);
+  return join.predicate === void 0 ? await broadcastJoin(leftRows, rightRows, {
     leftKey: join.leftKey,
     rightKey: join.rightKey,
     type: join.type,
     maxRightRows: options.joinMaxRightRows ?? 1e5
-  });
-  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
-  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
-  rows = projectRows(rows, ast);
-  if (ast.distinct === true) rows = distinctRows(rows);
-  return offsetLimitRows(rows, ast);
+  }) : predicateSubqueryJoinRows(leftRows, rightRows, join, options);
+}
+async function subqueryJoinRightRows(lake, join) {
+  if (subqueryJoinHasAggregation(join)) {
+    return await aggregateRowsFromAst(lake.path(join.source), {
+      source: join.source,
+      select: join.rightKey,
+      ...join.where === void 0 ? {} : { where: join.where },
+      ...join.groupBy === void 0 ? {} : { groupBy: join.groupBy },
+      ...join.aggregates === void 0 ? {} : { aggregates: join.aggregates },
+      ...join.hiddenAggregates === void 0 ? {} : { hiddenAggregates: join.hiddenAggregates },
+      ...join.having === void 0 ? {} : { having: join.having },
+      ...join.orderBy === void 0 ? {} : { orderBy: join.orderBy }
+    });
+  }
+  let rows = await lake.path(join.source).toArray();
+  if (join.where !== void 0) rows = rows.filter((row) => matches(join.where, row));
+  if (join.orderBy !== void 0) rows = sortRows(rows, join.orderBy);
+  return rows;
+}
+function subqueryJoinHasAggregation(join) {
+  return join.aggregates !== void 0 || join.having !== void 0 || join.groupBy !== void 0 && join.groupBy.length > 0;
+}
+function predicateSubqueryJoinRows(leftRows, rightRows, join, options) {
+  if (join.predicate === void 0) return leftRows;
+  const maxRightRows = options.joinMaxRightRows ?? 1e5;
+  if (rightRows.length > maxRightRows) {
+    throw new LakeqlError(
+      "LAKEQL_BUDGET_EXCEEDED",
+      `Predicate subquery join exceeded maxRightRows (${rightRows.length} > ${maxRightRows})`,
+      { metric: "maxRightRows", limit: maxRightRows, actual: rightRows.length }
+    );
+  }
+  const maxOutputRows = options.joinMaxOutputRows ?? 1e5;
+  const out = [];
+  const leftAlias = join.leftAlias ?? "";
+  const rightAlias = join.alias ?? join.source;
+  let candidateRows = 0;
+  for (const leftRow of leftRows) {
+    let matched = false;
+    for (const rightRow of rightRows) {
+      candidateRows += 1;
+      if (candidateRows > maxOutputRows) {
+        throw new LakeqlError(
+          "LAKEQL_BUDGET_EXCEEDED",
+          `Predicate subquery join exceeded maxOutputRows (${candidateRows} > ${maxOutputRows})`,
+          { metric: "maxOutputRows", limit: maxOutputRows, actual: candidateRows }
+        );
+      }
+      if (!subqueryKeysMatch(leftRow, rightRow, join)) continue;
+      const row = {
+        ...leftRow,
+        ...leftAlias.length === 0 ? {} : qualifyOnlyRow(leftRow, leftAlias),
+        ...qualifyOnlyRow(rightRow, rightAlias)
+      };
+      if (!matches(join.predicate, row)) continue;
+      matched = true;
+      break;
+    }
+    if (join.type === "semi" && matched || join.type === "anti" && !matched) out.push(leftRow);
+  }
+  return out;
+}
+function subqueryKeysMatch(leftRow, rightRow, join) {
+  for (const [index, leftKey] of join.leftKey.entries()) {
+    const rightKey = join.rightKey[index];
+    if (rightKey === void 0) return false;
+    const left = leftRow[leftKey];
+    const right = rightRow[rightKey];
+    if (left === null || right === null || left !== right) return false;
+  }
+  return true;
+}
+function offsetLimitSubqueryRows(rows, subquery) {
+  const offset = subquery.offset ?? 0;
+  if (subquery.limit !== void 0) return rows.slice(offset, offset + subquery.limit);
+  return offset > 0 ? rows.slice(offset) : rows;
 }
 function qualifyRow(row, alias) {
   const out = { ...row };
@@ -2541,7 +3498,19 @@ function qualifyRow(row, alias) {
 function qualifyOnlyRow(row, alias) {
   return Object.fromEntries(Object.entries(row).map(([key, value]) => [`${alias}.${key}`, value]));
 }
-function projectRows(rows, ast) {
+async function correlatedScalarRowsFromAst(lake, ast) {
+  let rows = await lake.path(ast.source).toArray();
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy, ast);
+  rows = await projectRows(lake, rows, ast);
+  if (ast.distinct === true) rows = distinctRows(rows);
+  return offsetLimitRows(rows, ast);
+}
+function hasCorrelatedScalarSubqueries(ast) {
+  return ast.correlatedScalarSubqueries !== void 0 && Object.keys(ast.correlatedScalarSubqueries).length > 0;
+}
+async function projectRows(lake, rows, ast) {
+  const lookups = await correlatedScalarLookups(lake, ast);
   const hasWildcardProjection = ast.select?.includes("*") ?? false;
   return rows.map((row) => {
     if (hasWildcardProjection && ast.projections === void 0) return row;
@@ -2555,11 +3524,123 @@ function projectRows(rows, ast) {
       }
     }
     for (const [alias, expr] of Object.entries(ast.projections ?? {})) {
-      out[alias] = evaluate(expr, row);
+      out[alias] = evaluate(resolveCorrelatedScalarExpr(expr, row, lookups), row);
     }
     return out;
   });
 }
+async function correlatedScalarLookups(lake, ast) {
+  const lookups = /* @__PURE__ */ new Map();
+  for (const [id, subquery] of Object.entries(ast.correlatedScalarSubqueries ?? {})) {
+    const rightRows = await aggregateRowsFromAst(lake.path(subquery.source), {
+      source: subquery.source,
+      select: subquery.rightKey,
+      groupBy: subquery.groupBy,
+      aggregates: subquery.aggregates,
+      ...subquery.hiddenAggregates === void 0 ? {} : { hiddenAggregates: subquery.hiddenAggregates },
+      ...subquery.where === void 0 ? {} : { where: subquery.where },
+      ...subquery.having === void 0 ? {} : { having: subquery.having }
+    });
+    lookups.set(id, {
+      subquery,
+      rows: new Map(rightRows.map((row) => [correlatedScalarRightKey(row, subquery), row]))
+    });
+  }
+  return lookups;
+}
+function resolveCorrelatedScalarExpr(expr, row, lookups) {
+  switch (expr.kind) {
+    case "call":
+      if (expr.fn === "__lakeql_correlated_scalar_subquery") {
+        return {
+          kind: "literal",
+          value: correlatedScalarValue(expr, row, lookups)
+        };
+      }
+      return {
+        ...expr,
+        args: expr.args.map((arg) => resolveCorrelatedScalarExpr(arg, row, lookups))
+      };
+    case "compare":
+      return {
+        ...expr,
+        left: resolveCorrelatedScalarExpr(expr.left, row, lookups),
+        right: resolveCorrelatedScalarExpr(expr.right, row, lookups)
+      };
+    case "arithmetic":
+      return {
+        ...expr,
+        left: resolveCorrelatedScalarExpr(expr.left, row, lookups),
+        right: resolveCorrelatedScalarExpr(expr.right, row, lookups)
+      };
+    case "case":
+      return {
+        ...expr,
+        whens: expr.whens.map((branch) => ({
+          when: resolveCorrelatedScalarExpr(branch.when, row, lookups),
+          value: resolveCorrelatedScalarExpr(branch.value, row, lookups)
+        })),
+        ...expr.else === void 0 ? {} : { else: resolveCorrelatedScalarExpr(expr.else, row, lookups) }
+      };
+    case "logical":
+      return {
+        ...expr,
+        operands: expr.operands.map(
+          (operand) => resolveCorrelatedScalarExpr(operand, row, lookups)
+        )
+      };
+    case "not":
+      return { ...expr, operand: resolveCorrelatedScalarExpr(expr.operand, row, lookups) };
+    case "between":
+      return {
+        ...expr,
+        target: resolveCorrelatedScalarExpr(expr.target, row, lookups),
+        low: resolveCorrelatedScalarExpr(expr.low, row, lookups),
+        high: resolveCorrelatedScalarExpr(expr.high, row, lookups)
+      };
+    case "in":
+      return {
+        ...expr,
+        target: resolveCorrelatedScalarExpr(expr.target, row, lookups),
+        values: expr.values.map((value) => resolveCorrelatedScalarExpr(value, row, lookups))
+      };
+    case "like":
+      return { ...expr, target: resolveCorrelatedScalarExpr(expr.target, row, lookups) };
+    case "null-check":
+      return { ...expr, target: resolveCorrelatedScalarExpr(expr.target, row, lookups) };
+    case "column":
+    case "literal":
+      return expr;
+  }
+}
+function correlatedScalarValue(expr, row, lookups) {
+  const id = expr.args[0];
+  if (id?.kind !== "literal" || typeof id.value !== "string") {
+    throw new LakeqlError(
+      "LAKEQL_SQL_UNSUPPORTED",
+      "Invalid correlated scalar subquery placeholder"
+    );
+  }
+  const lookup = lookups.get(id.value);
+  if (lookup === void 0) {
+    throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Missing correlated scalar subquery metadata");
+  }
+  const key = correlatedScalarLeftKey(row, lookup.subquery);
+  if (key === void 0) return null;
+  return lookup.rows.get(key)?.[lookup.subquery.column] ?? null;
+}
+function correlatedScalarLeftKey(row, subquery) {
+  const values = [];
+  for (const key of subquery.leftKey) {
+    const value = row[key];
+    if (value === null || value === void 0) return void 0;
+    values.push(value);
+  }
+  return JSON.stringify(values);
+}
+function correlatedScalarRightKey(row, subquery) {
+  return JSON.stringify(subquery.rightKey.map((key) => row[key]));
+}
 async function aggregateRowsFromAst(builder, ast) {
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over aggregates is not supported");
@@ -2567,6 +3648,8 @@ async function aggregateRowsFromAst(builder, ast) {
   validateAggregateProjection(ast);
   let next = builder;
   if (ast.where) next = next.where(ast.where);
+  const preProjections = aggregatePreProjections(ast);
+  if (preProjections !== void 0) next = next.select(["*"]).project(preProjections);
   const aggregates = ast.aggregates ?? {};
   const hiddenCountAlias = hiddenAggregateAlias(ast);
   const aggregateSpec = Object.keys(aggregates).length > 0 ? aggregates : { [hiddenCountAlias]: { op: "count" } };
@@ -2596,12 +3679,13 @@ function validateAggregateProjection(ast) {
 }
 function projectAggregateRows(rows, ast, hiddenCountAlias) {
   const aggregates = Object.entries(ast.aggregates ?? {});
+  const hiddenAggregates = new Set(ast.hiddenAggregates ?? []);
   const hasWildcardProjection = ast.select?.includes("*") ?? false;
   return rows.map((row) => {
     const out = {};
     if (hasWildcardProjection) {
       for (const [column, value] of Object.entries(row)) {
-        if (column !== hiddenCountAlias) out[column] = value;
+        if (column !== hiddenCountAlias && !hiddenAggregates.has(column)) out[column] = value;
       }
     } else if (ast.select !== void 0) {
       for (const select of ast.select ?? []) {
@@ -2609,13 +3693,24 @@ function projectAggregateRows(rows, ast, hiddenCountAlias) {
         out[alias] = row[column];
       }
     }
-    for (const [alias] of aggregates) out[alias] = row[alias];
+    for (const [alias] of aggregates) {
+      if (!hiddenAggregates.has(alias)) out[alias] = row[alias];
+    }
     for (const [alias, expr] of Object.entries(ast.projections ?? {})) {
-      out[alias] = evaluate(expr, row);
+      out[alias] = aggregateProjectionValue(alias, expr, row, ast);
     }
     return out;
   });
 }
+function aggregatePreProjections(ast) {
+  const groupBy = new Set(ast.groupBy ?? []);
+  const entries = Object.entries(ast.projections ?? {}).filter(([alias]) => groupBy.has(alias));
+  return entries.length === 0 ? void 0 : Object.fromEntries(entries);
+}
+function aggregateProjectionValue(alias, expr, row, ast) {
+  if (ast.groupBy?.includes(alias) && alias in row) return row[alias];
+  return evaluate(expr, row);
+}
 function hiddenAggregateAlias(ast) {
   const reserved = /* @__PURE__ */ new Set([...ast.select ?? [], ...Object.keys(ast.aggregates ?? {})]);
   let alias = "__lakeql_group_count";
@@ -2639,20 +3734,27 @@ function distinctRows(rows) {
   }
   return out;
 }
-function sortRows(rows, orderBy) {
+function sortRows(rows, orderBy, ast) {
   return [...rows].sort((a, b) => {
     for (const term of orderBy) {
-      const av = a[term.column] ?? null;
-      const bv = b[term.column] ?? null;
+      const av = sortValue(a, term.column, ast);
+      const bv = sortValue(b, term.column, ast);
       if (av === bv) continue;
-      if (av === null) return term.nulls === "first" ? -1 : 1;
-      if (bv === null) return term.nulls === "first" ? 1 : -1;
+      if (av === null || av === void 0) return term.nulls === "first" ? -1 : 1;
+      if (bv === null || bv === void 0) return term.nulls === "first" ? 1 : -1;
       const direction = term.direction === "desc" ? -1 : 1;
       return (av < bv ? -1 : 1) * direction;
     }
     return 0;
   });
 }
+function sortValue(row, column, ast) {
+  const expr = ast?.projections?.[column];
+  if (expr !== void 0) return evaluate(expr, row) ?? null;
+  const select = ast?.select?.find((value) => selectColumn(value).alias === column);
+  if (select !== void 0) return row[selectColumn(select).column] ?? null;
+  return row[column] ?? null;
+}
 function offsetLimitRows(rows, ast) {
   const offset = ast.offset ?? 0;
   if (ast.limit !== void 0) return rows.slice(offset, offset + ast.limit);