lakeql 0.2.0 → 0.5.1

package/dist/bin.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { createParquetLake, readParquetMetadata, memoryStore, parseSql, writePartitionedParquet, broadcastJoin, partitionedParquetOutputEntries, parseSqlStatement } from './chunk-32PE6IK4.js';
+import { createParquetLake, readParquetMetadata, memoryStore, parseSql, writePartitionedParquet, broadcastJoin, crossJoin, partitionedParquetOutputEntries, parseSqlStatement } from './chunk-AUL4DYT4.js';
 import { createOutputManifest, fingerprint, writeOutputManifest } from './chunk-LWF5FKHB.js';
 import { LakeqlError, matches, evaluate } from './chunk-WGHR35QU.js';
 import { readFile, mkdir, writeFile } from 'fs/promises';
@@ -47,7 +47,7 @@ function usage() {
     "commands:",
     "  compact --path <file.parquet> --output <prefix> [--max-rows-per-file n]",
     "  query   --path <file.parquet> --sql <query> [--format csv|json|ndjson]",
-    "  query   --table name=file.parquet [--table name=file.parquet ...] --sql <join-query> [--join-max-right-rows n]",
+    "  query   --table name=file.parquet [--table name=file.parquet ...] --sql <join-query> [--join-max-right-rows n] [--join-max-output-rows n]",
     "  explain --path <file.parquet> --sql <query>",
     "  inspect --path <file.parquet>",
     "  write   --path <file.parquet> --sql <query> --output <prefix> [--partition-by a,b] [--max-rows-per-file n] [--manifest <path>] [--job-id id]",
@@ -70,21 +70,20 @@ async function query(args) {
   const executableAst = await materializeCteIfNeeded(store, lake, ast);
   const executableLake = createParquetLake({ store });
   const resolvedAst = await resolveScalarSubqueries(executableLake, executableAst);
-  if (resolvedAst.subqueryJoin !== void 0) {
+  if (sqlSubqueryJoins(resolvedAst).length > 0) {
     const rows = await subqueryJoinRowsFromAst(executableLake, resolvedAst, args);
     if (args.format === "json") return `${jsonStringify(rows)}
 `;
     if (args.format === "csv") return rowsToCsv(rows);
     return rows.map((row) => jsonStringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
   }
-  if (resolvedAst.join !== void 0) {
+  if (sqlJoins(resolvedAst).length > 0) {
     const rows = await joinRowsFromAst(executableLake, resolvedAst, args);
     if (args.format === "json") return `${jsonStringify(rows)}
 `;
     if (args.format === "csv") return rowsToCsv(rows);
     return rows.map((row) => jsonStringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
   }
-  const result2 = builderFromAst(executableLake.path(resolvedAst.source), resolvedAst);
   if (hasAggregation(resolvedAst)) {
     const rows = await aggregateRowsFromAst(executableLake.path(resolvedAst.source), resolvedAst);
     if (args.format === "json") return `${jsonStringify(rows)}
@@ -92,6 +91,14 @@ async function query(args) {
     if (args.format === "csv") return rowsToCsv(rows);
     return rows.map((row) => jsonStringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
   }
+  if (hasCorrelatedScalarSubqueries(resolvedAst)) {
+    const rows = await correlatedScalarRowsFromAst(executableLake, resolvedAst);
+    if (args.format === "json") return `${jsonStringify(rows)}
+`;
+    if (args.format === "csv") return rowsToCsv(rows);
+    return rows.map((row) => jsonStringify(row)).join("\n") + (rows.length > 0 ? "\n" : "");
+  }
+  const result2 = builderFromAst(executableLake.path(resolvedAst.source), resolvedAst);
   if (args.format === "json") return `${jsonStringify(await result2.toArray())}
 `;
   if (args.format === "csv") return new Response(result2.streamCsv()).text();
@@ -147,7 +154,7 @@ async function write(args) {
   const executableAst = await materializeCteIfNeeded(store, lake, ast);
   const executableLake = createParquetLake({ store });
   const resolvedAst = await resolveScalarSubqueries(executableLake, executableAst);
-  const rows = resolvedAst.subqueryJoin !== void 0 ? await subqueryJoinRowsFromAst(executableLake, resolvedAst, args) : resolvedAst.join !== void 0 ? await joinRowsFromAst(executableLake, resolvedAst, args) : hasAggregation(resolvedAst) ? await aggregateRowsFromAst(executableLake.path(resolvedAst.source), resolvedAst) : await builderFromAst(executableLake.path(resolvedAst.source), resolvedAst).toArray();
+  const rows = sqlSubqueryJoins(resolvedAst).length > 0 ? await subqueryJoinRowsFromAst(executableLake, resolvedAst, args) : sqlJoins(resolvedAst).length > 0 ? await joinRowsFromAst(executableLake, resolvedAst, args) : hasAggregation(resolvedAst) ? await aggregateRowsFromAst(executableLake.path(resolvedAst.source), resolvedAst) : hasCorrelatedScalarSubqueries(resolvedAst) ? await correlatedScalarRowsFromAst(executableLake, resolvedAst) : await builderFromAst(executableLake.path(resolvedAst.source), resolvedAst).toArray();
   return writeRows(outputPrefix, rows, args);
 }
 async function writeRows(outputPrefix, rows, args) {
@@ -226,7 +233,7 @@ async function materializeCteIfNeeded(store, lake, ast) {
       "CTEs are only supported as the outer FROM source"
     );
   }
-  if (ast.join !== void 0 || ast.subqueryJoin !== void 0) {
+  if (sqlJoins(ast).length > 0 || sqlSubqueryJoins(ast).length > 0) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "CTEs inside JOINs are not supported yet");
   }
   const cteRows = hasAggregation(ast.cte.query) ? await aggregateRowsFromAst(lake.path(ast.cte.query.source), ast.cte.query) : await builderFromAst(lake.path(ast.cte.query.source), ast.cte.query).toArray();
@@ -246,6 +253,10 @@ async function resolveScalarSubqueries(lake, ast) {
   const values = /* @__PURE__ */ new Map();
   for (const [id, subquery] of Object.entries(ast.scalarSubqueries)) {
     const rows = hasAggregation(subquery.query) ? await aggregateRowsFromAst(lake.path(subquery.query.source), subquery.query) : await builderFromAst(lake.path(subquery.query.source), subquery.query).toArray();
+    if (subquery.mode === "exists") {
+      values.set(id, rows.length > 0);
+      continue;
+    }
     if (rows.length > 1) {
       throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Scalar subquery returned more than one row");
     }
@@ -262,6 +273,21 @@ async function resolveScalarSubqueries(lake, ast) {
       ])
     );
   }
+  if (ast.subqueryJoins !== void 0) {
+    out.subqueryJoins = ast.subqueryJoins.map((join) => ({
+      ...join,
+      ...join.where === void 0 ? {} : { where: replaceScalarSubqueryExpr(join.where, values) },
+      ...join.predicate === void 0 ? {} : { predicate: replaceScalarSubqueryExpr(join.predicate, values) }
+    }));
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  } else if (ast.subqueryJoin?.where !== void 0 || ast.subqueryJoin?.predicate !== void 0) {
+    out.subqueryJoin = {
+      ...ast.subqueryJoin,
+      ...ast.subqueryJoin.where === void 0 ? {} : { where: replaceScalarSubqueryExpr(ast.subqueryJoin.where, values) },
+      ...ast.subqueryJoin.predicate === void 0 ? {} : { predicate: replaceScalarSubqueryExpr(ast.subqueryJoin.predicate, values) }
+    };
+  }
   delete out.scalarSubqueries;
   return out;
 }
@@ -327,81 +353,91 @@ function replaceScalarSubqueryExpr(expr, values) {
   }
 }
 async function joinRowsFromAst(lake, ast, args) {
-  if (ast.join === void 0) throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
-  const join = ast.join;
+  const joins = sqlJoins(ast);
+  const firstJoin = joins[0];
+  if (firstJoin === void 0) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
+  }
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over JOIN is not supported");
   }
   if (hasAggregation(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Aggregate SQL over JOIN is not supported yet");
   }
-  const leftAlias = leftJoinAlias(ast);
-  const plan = planJoinSides(ast, leftAlias, join.alias);
-  let leftBuilder = lake.path(ast.source);
-  let rightBuilder = lake.path(join.source);
-  if (plan.leftWhere !== void 0) leftBuilder = leftBuilder.where(plan.leftWhere);
-  if (plan.rightWhere !== void 0) rightBuilder = rightBuilder.where(plan.rightWhere);
-  if (plan.leftColumns !== void 0) leftBuilder = leftBuilder.select(plan.leftColumns);
-  if (plan.rightColumns !== void 0) rightBuilder = rightBuilder.select(plan.rightColumns);
-  const leftRows = (await leftBuilder.toArray()).map((row) => qualifyRow(row, leftAlias));
-  const rightRows = (await rightBuilder.toArray()).map((row) => qualifyRow(row, join.alias));
-  let rows = await broadcastJoin(leftRows, rightRows, {
-    leftKey: join.leftKey,
-    rightKey: join.rightKey,
-    type: join.type,
-    rightPrefix: `${join.alias}.`,
-    maxRightRows: args.joinMaxRightRows ?? 1e5
-  });
-  if (join.type === "left") {
-    rows = fillLeftJoinNulls(rows, join.alias, leftJoinRightColumns(ast, join.alias, rightRows));
+  let rows = (await lake.path(ast.source).toArray()).map(
+    (row) => qualifyRow(row, firstJoin.leftAlias)
+  );
+  for (const join of joins) {
+    const rightRows = (await lake.path(join.source).toArray()).map(
+      (row) => qualifyRow(row, join.alias)
+    );
+    if (join.type === "right") {
+      const leftRows = rows;
+      rows = await broadcastJoin(rightRows, leftRows, {
+        leftKey: join.rightKey,
+        rightKey: join.leftKey,
+        type: "left",
+        rightPrefix: `${join.leftAlias}.`,
+        maxRightRows: args.joinMaxRightRows ?? 1e5
+      });
+      rows = fillRightJoinNulls(rows, ast, join.alias, leftRows);
+    } else if (join.type === "full") {
+      const leftRows = rows;
+      const leftJoined = await broadcastJoin(leftRows, rightRows, {
+        leftKey: join.leftKey,
+        rightKey: join.rightKey,
+        type: "left",
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: args.joinMaxRightRows ?? 1e5
+      });
+      const unmatchedRight = await broadcastJoin(rightRows, leftRows, {
+        leftKey: join.rightKey,
+        rightKey: join.leftKey,
+        type: "anti",
+        rightPrefix: `${join.leftAlias}.`,
+        maxRightRows: args.joinMaxRightRows ?? 1e5
+      });
+      rows = [
+        ...fillLeftJoinNulls(
+          leftJoined,
+          join.alias,
+          leftJoinRightColumns(ast, join.alias, rightRows)
+        ),
+        ...fillRightJoinNulls(unmatchedRight, ast, join.alias, leftRows)
+      ];
+    } else {
+      rows = join.type === "cross" ? await crossJoin(rows, rightRows, {
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: args.joinMaxRightRows ?? 1e5,
+        maxOutputRows: args.joinMaxOutputRows ?? 1e5
+      }) : await broadcastJoin(rows, rightRows, {
+        leftKey: join.leftKey,
+        rightKey: join.rightKey,
+        type: join.type,
+        rightPrefix: `${join.alias}.`,
+        maxRightRows: args.joinMaxRightRows ?? 1e5
+      });
+    }
+    if (join.type === "left") {
+      rows = fillLeftJoinNulls(rows, join.alias, leftJoinRightColumns(ast, join.alias, rightRows));
+    }
   }
-  if (plan.residualWhere !== void 0)
-    rows = rows.filter((row) => matches(plan.residualWhere, row));
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
   if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
-  rows = projectRows(rows, ast);
+  rows = await projectRows(lake, rows, ast);
   if (ast.distinct === true) rows = distinctRows(rows);
   const offset = ast.offset ?? 0;
   if (ast.limit !== void 0) rows = rows.slice(offset, offset + ast.limit);
   else if (offset > 0) rows = rows.slice(offset);
   return rows;
 }
-function planJoinSides(ast, leftAlias, rightAlias) {
-  if (ast.join === void 0) throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL JOIN");
-  const leftPredicates = [];
-  const rightPredicates = [];
-  const residualPredicates = [];
-  for (const predicate of splitAndPredicate(ast.where)) {
-    const columns = exprColumns(predicate);
-    if (columns.length > 0 && columns.every((column) => isQualifiedBy(column, leftAlias))) {
-      leftPredicates.push(stripQualifiedExpr(predicate, leftAlias));
-    } else if (ast.join.type === "inner" && columns.length > 0 && columns.every((column) => isQualifiedBy(column, rightAlias))) {
-      rightPredicates.push(stripQualifiedExpr(predicate, rightAlias));
-    } else {
-      residualPredicates.push(predicate);
-    }
-  }
-  const plan = {};
-  const leftWhere = combineAndPredicate(leftPredicates);
-  const rightWhere = combineAndPredicate(rightPredicates);
-  const residualWhere = combineAndPredicate(residualPredicates);
-  if (leftWhere !== void 0) plan.leftWhere = leftWhere;
-  if (rightWhere !== void 0) plan.rightWhere = rightWhere;
-  if (residualWhere !== void 0) plan.residualWhere = residualWhere;
-  if (canProjectJoinSides(ast, leftAlias, rightAlias)) {
-    plan.leftColumns = joinSideColumns(ast, leftAlias, ast.join.leftKey);
-    plan.rightColumns = joinSideColumns(ast, rightAlias, ast.join.rightKey);
-  }
-  return plan;
-}
-function splitAndPredicate(expr) {
-  if (expr === void 0) return [];
-  if (expr.kind === "logical" && expr.op === "and") return expr.operands.flatMap(splitAndPredicate);
-  return [expr];
-}
-function combineAndPredicate(predicates) {
-  if (predicates.length === 0) return void 0;
-  if (predicates.length === 1) return predicates[0];
-  return { kind: "logical", op: "and", operands: predicates };
+function sqlJoins(ast) {
+  if (ast.joins !== void 0) return ast.joins;
+  return ast.join === void 0 ? [] : [ast.join];
+}
+function sqlSubqueryJoins(ast) {
+  if (ast.subqueryJoins !== void 0) return ast.subqueryJoins;
+  return ast.subqueryJoin === void 0 ? [] : [ast.subqueryJoin];
 }
 function isQualifiedBy(column, alias) {
   return column.startsWith(`${alias}.`) && column.length > alias.length + 1;
@@ -409,66 +445,6 @@ function isQualifiedBy(column, alias) {
 function stripQualifiedColumn(column, alias) {
   return isQualifiedBy(column, alias) ? column.slice(alias.length + 1) : column;
 }
-function stripQualifiedExpr(expr, alias) {
-  switch (expr.kind) {
-    case "column":
-      return { ...expr, name: stripQualifiedColumn(expr.name, alias) };
-    case "compare":
-      return {
-        ...expr,
-        left: stripQualifiedExpr(expr.left, alias),
-        right: stripQualifiedExpr(expr.right, alias)
-      };
-    case "between":
-      return {
-        ...expr,
-        target: stripQualifiedExpr(expr.target, alias),
-        low: stripQualifiedExpr(expr.low, alias),
-        high: stripQualifiedExpr(expr.high, alias)
-      };
-    case "in":
-      return {
-        ...expr,
-        target: stripQualifiedExpr(expr.target, alias),
-        values: expr.values.map((value) => stripQualifiedExpr(value, alias))
-      };
-    case "logical":
-      return {
-        ...expr,
-        operands: expr.operands.map((operand) => stripQualifiedExpr(operand, alias))
-      };
-    case "not":
-      return { ...expr, operand: stripQualifiedExpr(expr.operand, alias) };
-    case "null-check":
-      return { ...expr, target: stripQualifiedExpr(expr.target, alias) };
-    case "like":
-      return { ...expr, target: stripQualifiedExpr(expr.target, alias) };
-    case "call":
-      return { ...expr, args: expr.args.map((arg) => stripQualifiedExpr(arg, alias)) };
-    case "arithmetic":
-      return {
-        ...expr,
-        left: stripQualifiedExpr(expr.left, alias),
-        right: stripQualifiedExpr(expr.right, alias)
-      };
-    case "case":
-      return {
-        ...expr,
-        whens: expr.whens.map((branch) => ({
-          when: stripQualifiedExpr(branch.when, alias),
-          value: stripQualifiedExpr(branch.value, alias)
-        })),
-        ...expr.else === void 0 ? {} : { else: stripQualifiedExpr(expr.else, alias) }
-      };
-    case "literal":
-      return expr;
-  }
-}
-function exprColumns(expr) {
-  const columns = /* @__PURE__ */ new Set();
-  collectExprColumns(expr, columns);
-  return [...columns];
-}
 function collectExprColumns(expr, columns) {
   switch (expr.kind) {
     case "column":
@@ -517,12 +493,6 @@ function collectExprColumns(expr, columns) {
       return;
   }
 }
-function canProjectJoinSides(ast, leftAlias, rightAlias) {
-  if (ast.select?.includes("*") ?? false) return false;
-  return referencedJoinColumns(ast).every(
-    (column) => isQualifiedBy(column, leftAlias) || isQualifiedBy(column, rightAlias)
-  );
-}
 function referencedJoinColumns(ast) {
   const columns = /* @__PURE__ */ new Set();
   for (const select of ast.select ?? []) {
@@ -534,17 +504,6 @@ function referencedJoinColumns(ast) {
   for (const term of ast.orderBy ?? []) columns.add(term.column);
   return [...columns].filter((column) => column !== "*");
 }
-function joinSideColumns(ast, alias, joinKey) {
-  const columns = new Set(
-    (Array.isArray(joinKey) ? joinKey : [joinKey]).map(
-      (column) => stripQualifiedColumn(column, alias)
-    )
-  );
-  for (const column of referencedJoinColumns(ast)) {
-    if (isQualifiedBy(column, alias)) columns.add(stripQualifiedColumn(column, alias));
-  }
-  return [...columns];
-}
 function leftJoinRightColumns(ast, rightAlias, rightRows) {
   const columns = /* @__PURE__ */ new Set();
   if (ast.select?.includes("*") ?? false) {
@@ -573,47 +532,241 @@ function fillLeftJoinNulls(rows, rightAlias, rightColumns) {
     return out ?? row;
   });
 }
+function fillRightJoinNulls(rows, ast, preservedAlias, leftRows) {
+  const columns = rightJoinLeftColumns(ast, preservedAlias, leftRows);
+  if (columns.length === 0) return rows;
+  return rows.map((row) => {
+    let out;
+    for (const column of columns) {
+      if (column in row) continue;
+      out ??= { ...row };
+      out[column] = null;
+    }
+    return out ?? row;
+  });
+}
+function rightJoinLeftColumns(ast, preservedAlias, leftRows) {
+  const columns = /* @__PURE__ */ new Set();
+  if (ast.select?.includes("*") ?? false) {
+    for (const row of leftRows) {
+      for (const column of Object.keys(row)) {
+        if (!isQualifiedBy(column, preservedAlias)) columns.add(column);
+      }
+    }
+  }
+  for (const column of referencedJoinColumns(ast)) {
+    if (!isQualifiedBy(column, preservedAlias)) columns.add(column);
+  }
+  return [...columns];
+}
 async function subqueryJoinRowsFromAst(lake, ast, args) {
-  if (ast.subqueryJoin === void 0) {
+  const joins = sqlSubqueryJoins(ast);
+  if (joins.length === 0) {
     throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Missing SQL IN subquery");
   }
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over IN subquery is not supported");
   }
   if (hasAggregation(ast)) {
-    throw new LakeqlError(
-      "LAKEQL_SQL_UNSUPPORTED",
-      "Aggregate SQL over IN subquery is not supported yet"
-    );
+    return await aggregateSubqueryJoinRowsFromAst(lake, ast, joins, args);
+  }
+  let rows = await lake.path(ast.source).toArray();
+  for (const join of joins) {
+    rows = await applySubqueryJoinRows(lake, rows, join, args);
   }
-  const join = ast.subqueryJoin;
-  let rightRows = await lake.path(join.source).toArray();
-  if (join.where !== void 0) rightRows = rightRows.filter((row) => matches(join.where, row));
-  let rows = await broadcastJoin(await lake.path(ast.source).toArray(), rightRows, {
-    leftKey: join.leftKey,
-    rightKey: join.rightKey,
-    type: join.type,
-    maxRightRows: args.joinMaxRightRows ?? 1e5
-  });
   if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
   if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
-  rows = projectRows(rows, ast);
+  rows = await projectRows(lake, rows, ast);
   if (ast.distinct === true) rows = distinctRows(rows);
   const offset = ast.offset ?? 0;
   if (ast.limit !== void 0) rows = rows.slice(offset, offset + ast.limit);
   else if (offset > 0) rows = rows.slice(offset);
   return rows;
 }
-function leftJoinAlias(ast) {
-  if (ast.join === void 0) return ast.source;
-  return ast.join.leftAlias;
+async function aggregateSubqueryJoinRowsFromAst(lake, ast, joins, args) {
+  let rows = await lake.path(ast.source).toArray();
+  for (const join of joins) rows = await applySubqueryJoinRows(lake, rows, join, args);
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
+  const aggregateAst = subqueryJoinAggregateAst(ast);
+  if (rows.length === 0) {
+    if ((ast.groupBy?.length ?? 0) > 0) return [];
+    return emptyAggregateRowsFromAst(aggregateAst);
+  }
+  const prefix = `__lakeql_subquery/${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+  const written = await writePartitionedParquet(lake.store, prefix, {
+    rows,
+    maxRowsPerFile: rows.length
+  });
+  try {
+    return await aggregateRowsFromAst(lake.path(`${prefix}/*.parquet`), {
+      ...aggregateAst,
+      source: `${prefix}/*.parquet`
+    });
+  } finally {
+    await Promise.all(written.files.map((file) => lake.store.delete(file.path)));
+  }
+}
+function subqueryJoinAggregateAst(ast) {
+  const {
+    subqueryJoin: _subqueryJoin,
+    subqueryJoins: _subqueryJoins,
+    where: _where,
+    ...rest
+  } = ast;
+  return rest;
+}
+function emptyAggregateRowsFromAst(ast) {
+  validateAggregateProjection(ast);
+  const hiddenCountAlias = hiddenAggregateAlias(ast);
+  const aggregateSpec = ast.aggregates === void 0 || Object.keys(ast.aggregates).length === 0 ? { [hiddenCountAlias]: { op: "count" } } : ast.aggregates;
+  let rows = [emptyAggregateRow(aggregateSpec)];
+  if (ast.having !== void 0) rows = rows.filter((row) => matches(ast.having, row));
+  rows = projectAggregateRows(rows, ast, hiddenCountAlias);
+  if (ast.distinct === true) rows = distinctRows(rows);
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
+  const offset = ast.offset ?? 0;
+  if (ast.limit !== void 0) rows = rows.slice(offset, offset + ast.limit);
+  else if (offset > 0) rows = rows.slice(offset);
+  return rows;
+}
+function emptyAggregateRow(spec) {
+  return Object.fromEntries(
+    Object.entries(spec).map(([alias, aggregate]) => [alias, emptyAggregateValue(aggregate.op)])
+  );
+}
+function emptyAggregateValue(op) {
+  switch (op) {
+    case "count":
+    case "count_distinct":
+    case "approx_count_distinct":
+      return 0;
+    case "sum":
+    case "avg":
+    case "var_samp":
+    case "var_pop":
+    case "stddev_samp":
+    case "stddev_pop":
+    case "median":
+    case "quantile":
+    case "min":
+    case "max":
+    case "mode":
+    case "first":
+    case "last":
+    case "any":
+      return null;
+  }
+}
+async function applySubqueryJoinRows(lake, leftRows, join, args) {
+  let rightRows = await subqueryJoinRightRows(lake, join);
+  rightRows = offsetLimitSubqueryRows(rightRows, join);
+  return join.predicate === void 0 ? await broadcastJoin(leftRows, rightRows, {
+    leftKey: join.leftKey,
+    rightKey: join.rightKey,
+    type: join.type,
+    maxRightRows: args.joinMaxRightRows ?? 1e5
+  }) : predicateSubqueryJoinRows(leftRows, rightRows, join, args);
+}
+async function subqueryJoinRightRows(lake, join) {
+  if (subqueryJoinHasAggregation(join)) {
+    return await aggregateRowsFromAst(lake.path(join.source), {
+      source: join.source,
+      select: join.rightKey,
+      ...join.where === void 0 ? {} : { where: join.where },
+      ...join.groupBy === void 0 ? {} : { groupBy: join.groupBy },
+      ...join.aggregates === void 0 ? {} : { aggregates: join.aggregates },
+      ...join.hiddenAggregates === void 0 ? {} : { hiddenAggregates: join.hiddenAggregates },
+      ...join.having === void 0 ? {} : { having: join.having },
+      ...join.orderBy === void 0 ? {} : { orderBy: join.orderBy }
+    });
+  }
+  let rows = await lake.path(join.source).toArray();
+  if (join.where !== void 0) rows = rows.filter((row) => matches(join.where, row));
+  if (join.orderBy !== void 0) rows = sortRows(rows, join.orderBy);
+  return rows;
+}
+function subqueryJoinHasAggregation(join) {
+  return join.aggregates !== void 0 || join.having !== void 0 || join.groupBy !== void 0 && join.groupBy.length > 0;
+}
+function predicateSubqueryJoinRows(leftRows, rightRows, join, args) {
+  if (join.predicate === void 0) return leftRows;
+  const maxRightRows = args.joinMaxRightRows ?? 1e5;
+  if (rightRows.length > maxRightRows) {
+    throw new LakeqlError(
+      "LAKEQL_BUDGET_EXCEEDED",
+      `Predicate subquery join exceeded maxRightRows (${rightRows.length} > ${maxRightRows})`,
+      { metric: "maxRightRows", limit: maxRightRows, actual: rightRows.length }
+    );
+  }
+  const maxOutputRows = args.joinMaxOutputRows ?? 1e5;
+  const out = [];
+  const leftAlias = join.leftAlias ?? "";
+  const rightAlias = join.alias ?? join.source;
+  let candidateRows = 0;
+  for (const leftRow of leftRows) {
+    let matched = false;
+    for (const rightRow of rightRows) {
+      candidateRows += 1;
+      if (candidateRows > maxOutputRows) {
+        throw new LakeqlError(
+          "LAKEQL_BUDGET_EXCEEDED",
+          `Predicate subquery join exceeded maxOutputRows (${candidateRows} > ${maxOutputRows})`,
+          { metric: "maxOutputRows", limit: maxOutputRows, actual: candidateRows }
+        );
+      }
+      if (!subqueryKeysMatch(leftRow, rightRow, join)) continue;
+      const row = {
+        ...leftRow,
+        ...leftAlias.length === 0 ? {} : qualifyOnlyRow(leftRow, leftAlias),
+        ...qualifyOnlyRow(rightRow, rightAlias)
+      };
+      if (!matches(join.predicate, row)) continue;
+      matched = true;
+      break;
+    }
+    if (join.type === "semi" && matched || join.type === "anti" && !matched) out.push(leftRow);
+  }
+  return out;
+}
+function subqueryKeysMatch(leftRow, rightRow, join) {
+  for (const [index, leftKey] of join.leftKey.entries()) {
+    const rightKey = join.rightKey[index];
+    if (rightKey === void 0) return false;
+    const left = leftRow[leftKey];
+    const right = rightRow[rightKey];
+    if (left === null || right === null || left !== right) return false;
+  }
+  return true;
+}
+function offsetLimitSubqueryRows(rows, subquery) {
+  const offset = subquery.offset ?? 0;
+  if (subquery.limit !== void 0) return rows.slice(offset, offset + subquery.limit);
+  return offset > 0 ? rows.slice(offset) : rows;
 }
 function qualifyRow(row, alias) {
   const out = { ...row };
   for (const [key, value] of Object.entries(row)) out[`${alias}.${key}`] = value;
   return out;
 }
-function projectRows(rows, ast) {
+function qualifyOnlyRow(row, alias) {
+  return Object.fromEntries(Object.entries(row).map(([key, value]) => [`${alias}.${key}`, value]));
+}
+async function correlatedScalarRowsFromAst(lake, ast) {
+  let rows = await lake.path(ast.source).toArray();
+  if (ast.where !== void 0) rows = rows.filter((row) => matches(ast.where, row));
+  if (ast.orderBy !== void 0) rows = sortRows(rows, ast.orderBy);
+  rows = await projectRows(lake, rows, ast);
+  if (ast.distinct === true) rows = distinctRows(rows);
+  const offset = ast.offset ?? 0;
+  if (ast.limit !== void 0) rows = rows.slice(offset, offset + ast.limit);
+  else if (offset > 0) rows = rows.slice(offset);
+  return rows;
+}
+function hasCorrelatedScalarSubqueries(ast) {
+  return ast.correlatedScalarSubqueries !== void 0 && Object.keys(ast.correlatedScalarSubqueries).length > 0;
+}
+async function projectRows(lake, rows, ast) {
+  const lookups = await correlatedScalarLookups(lake, ast);
   const hasWildcardProjection = ast.select?.includes("*") ?? false;
   return rows.map((row) => {
     if (hasWildcardProjection && ast.projections === void 0) return row;
@@ -627,11 +780,123 @@ function projectRows(rows, ast) {
       }
     }
     for (const [alias, expr] of Object.entries(ast.projections ?? {})) {
-      out[alias] = evaluate(expr, row);
+      out[alias] = evaluate(resolveCorrelatedScalarExpr(expr, row, lookups), row);
     }
     return out;
   });
 }
+async function correlatedScalarLookups(lake, ast) {
+  const lookups = /* @__PURE__ */ new Map();
+  for (const [id, subquery] of Object.entries(ast.correlatedScalarSubqueries ?? {})) {
+    const rightRows = await aggregateRowsFromAst(lake.path(subquery.source), {
+      source: subquery.source,
+      select: subquery.rightKey,
+      groupBy: subquery.groupBy,
+      aggregates: subquery.aggregates,
+      ...subquery.hiddenAggregates === void 0 ? {} : { hiddenAggregates: subquery.hiddenAggregates },
+      ...subquery.where === void 0 ? {} : { where: subquery.where },
+      ...subquery.having === void 0 ? {} : { having: subquery.having }
+    });
+    lookups.set(id, {
+      subquery,
+      rows: new Map(rightRows.map((row) => [correlatedScalarRightKey(row, subquery), row]))
+    });
+  }
+  return lookups;
+}
+function resolveCorrelatedScalarExpr(expr, row, lookups) {
+  switch (expr.kind) {
+    case "call":
+      if (expr.fn === "__lakeql_correlated_scalar_subquery") {
+        return {
+          kind: "literal",
+          value: correlatedScalarValue(expr, row, lookups)
+        };
+      }
+      return {
+        ...expr,
+        args: expr.args.map((arg) => resolveCorrelatedScalarExpr(arg, row, lookups))
+      };
+    case "compare":
+      return {
+        ...expr,
+        left: resolveCorrelatedScalarExpr(expr.left, row, lookups),
+        right: resolveCorrelatedScalarExpr(expr.right, row, lookups)
+      };
+    case "arithmetic":
+      return {
+        ...expr,
+        left: resolveCorrelatedScalarExpr(expr.left, row, lookups),
+        right: resolveCorrelatedScalarExpr(expr.right, row, lookups)
+      };
+    case "case":
+      return {
+        ...expr,
+        whens: expr.whens.map((branch) => ({
+          when: resolveCorrelatedScalarExpr(branch.when, row, lookups),
+          value: resolveCorrelatedScalarExpr(branch.value, row, lookups)
+        })),
+        ...expr.else === void 0 ? {} : { else: resolveCorrelatedScalarExpr(expr.else, row, lookups) }
+      };
+    case "logical":
+      return {
+        ...expr,
+        operands: expr.operands.map(
+          (operand) => resolveCorrelatedScalarExpr(operand, row, lookups)
+        )
+      };
+    case "not":
+      return { ...expr, operand: resolveCorrelatedScalarExpr(expr.operand, row, lookups) };
+    case "between":
+      return {
+        ...expr,
+        target: resolveCorrelatedScalarExpr(expr.target, row, lookups),
+        low: resolveCorrelatedScalarExpr(expr.low, row, lookups),
+        high: resolveCorrelatedScalarExpr(expr.high, row, lookups)
+      };
+    case "in":
+      return {
+        ...expr,
+        target: resolveCorrelatedScalarExpr(expr.target, row, lookups),
+        values: expr.values.map((value) => resolveCorrelatedScalarExpr(value, row, lookups))
+      };
+    case "like":
+      return { ...expr, target: resolveCorrelatedScalarExpr(expr.target, row, lookups) };
+    case "null-check":
+      return { ...expr, target: resolveCorrelatedScalarExpr(expr.target, row, lookups) };
+    case "column":
+    case "literal":
+      return expr;
+  }
+}
+function correlatedScalarValue(expr, row, lookups) {
+  const id = expr.args[0];
+  if (id?.kind !== "literal" || typeof id.value !== "string") {
+    throw new LakeqlError(
+      "LAKEQL_SQL_UNSUPPORTED",
+      "Invalid correlated scalar subquery placeholder"
+    );
+  }
+  const lookup = lookups.get(id.value);
+  if (lookup === void 0) {
+    throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Missing correlated scalar subquery metadata");
+  }
+  const key = correlatedScalarLeftKey(row, lookup.subquery);
+  if (key === void 0) return null;
+  return lookup.rows.get(key)?.[lookup.subquery.column] ?? null;
+}
+function correlatedScalarLeftKey(row, subquery) {
+  const values = [];
+  for (const key of subquery.leftKey) {
+    const value = row[key];
+    if (value === null || value === void 0) return void 0;
+    values.push(value);
+  }
+  return JSON.stringify(values);
+}
+function correlatedScalarRightKey(row, subquery) {
+  return JSON.stringify(subquery.rightKey.map((key) => row[key]));
+}
 async function aggregateRowsFromAst(builder, ast) {
   if (hasWindowSemantics(ast)) {
     throw new LakeqlError("LAKEQL_SQL_UNSUPPORTED", "Window SQL over aggregates is not supported");
@@ -639,6 +904,8 @@ async function aggregateRowsFromAst(builder, ast) {
   validateAggregateProjection(ast);
   let next = builder;
   if (ast.where) next = next.where(ast.where);
+  const preProjections = aggregatePreProjections(ast);
+  if (preProjections !== void 0) next = next.select(["*"]).project(preProjections);
   const aggregates = ast.aggregates ?? {};
   const hiddenCountAlias = hiddenAggregateAlias(ast);
   const aggregateSpec = Object.keys(aggregates).length > 0 ? aggregates : { [hiddenCountAlias]: { op: "count" } };
@@ -672,12 +939,13 @@ function validateAggregateProjection(ast) {
 }
 function projectAggregateRows(rows, ast, hiddenCountAlias) {
   const aggregates = Object.entries(ast.aggregates ?? {});
+  const hiddenAggregates = new Set(ast.hiddenAggregates ?? []);
   const hasWildcardProjection = ast.select?.includes("*") ?? false;
   return rows.map((row) => {
     const out = {};
     if (hasWildcardProjection) {
       for (const [column, value] of Object.entries(row)) {
-        if (column !== hiddenCountAlias) out[column] = value;
+        if (column !== hiddenCountAlias && !hiddenAggregates.has(column)) out[column] = value;
       }
     } else if (ast.select !== void 0) {
       for (const select of ast.select ?? []) {
@@ -685,13 +953,24 @@ function projectAggregateRows(rows, ast, hiddenCountAlias) {
         out[alias] = row[column];
       }
     }
-    for (const [alias] of aggregates) out[alias] = row[alias];
+    for (const [alias] of aggregates) {
+      if (!hiddenAggregates.has(alias)) out[alias] = row[alias];
+    }
     for (const [alias, expr] of Object.entries(ast.projections ?? {})) {
-      out[alias] = evaluate(expr, row);
+      out[alias] = aggregateProjectionValue(alias, expr, row, ast);
     }
     return out;
   });
 }
+function aggregatePreProjections(ast) {
+  const groupBy = new Set(ast.groupBy ?? []);
+  const entries = Object.entries(ast.projections ?? {}).filter(([alias]) => groupBy.has(alias));
+  return entries.length === 0 ? void 0 : Object.fromEntries(entries);
+}
+function aggregateProjectionValue(alias, expr, row, ast) {
+  if (ast.groupBy?.includes(alias) && alias in row) return row[alias];
+  return evaluate(expr, row);
+}
 function hiddenAggregateAlias(ast) {
   const reserved = /* @__PURE__ */ new Set([...ast.select ?? [], ...Object.keys(ast.aggregates ?? {})]);
   let alias = "__lakeql_group_count";
@@ -790,8 +1069,19 @@ function applyDefaultSource(ast, defaultSource, includeScalarSubqueries = true,
   } else {
     out.source = defaultSource;
   }
-  if (out.subqueryJoin !== void 0)
+  if (out.join !== void 0) out.join = { ...out.join, source: defaultSource };
+  if (out.joins !== void 0) {
+    out.joins = out.joins.map((join) => ({ ...join, source: defaultSource }));
+    const firstJoin = out.joins[0];
+    if (firstJoin !== void 0) out.join = firstJoin;
+  }
+  if (out.subqueryJoin !== void 0 && out.subqueryJoins === void 0)
     out.subqueryJoin = { ...out.subqueryJoin, source: defaultSource };
+  if (out.subqueryJoins !== void 0) {
+    out.subqueryJoins = out.subqueryJoins.map((join) => ({ ...join, source: defaultSource }));
+    const firstSubqueryJoin = out.subqueryJoins[0];
+    if (firstSubqueryJoin !== void 0) out.subqueryJoin = firstSubqueryJoin;
+  }
   if (includeScalarSubqueries && out.scalarSubqueries !== void 0) {
     out.scalarSubqueries = Object.fromEntries(
       Object.entries(out.scalarSubqueries).map(([id, subquery]) => [
@@ -800,6 +1090,14 @@ function applyDefaultSource(ast, defaultSource, includeScalarSubqueries = true,
       ])
     );
   }
+  if (out.correlatedScalarSubqueries !== void 0) {
+    out.correlatedScalarSubqueries = Object.fromEntries(
+      Object.entries(out.correlatedScalarSubqueries).map(([id, subquery]) => [
+        id,
+        { ...subquery, source: defaultSource }
+      ])
+    );
+  }
   return out;
 }
 function addDefaultFrom(sql) {
@@ -861,6 +1159,9 @@ function parseArgs(argv) {
     } else if (arg === "--join-max-right-rows") {
       index += 1;
       args.joinMaxRightRows = parsePositiveInt(requireValue(rest, index, arg), arg);
+    } else if (arg === "--join-max-output-rows") {
+      index += 1;
+      args.joinMaxOutputRows = parsePositiveInt(requireValue(rest, index, arg), arg);
     } else if (arg === "--manifest") {
       index += 1;
       args.manifest = requireValue(rest, index, arg);