@gscdump/engine 5.9.0 → 6.0.0

package/dist/adapters/hyparquet.mjs

@@ -1,5 +1,6 @@
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import { SCHEMAS } from "../schema.mjs";
+import { decompress } from "../node_modules/.pnpm/fzstd@0.1.1/node_modules/fzstd/esm/index.mjs";
 import { parquetMetadataAsync, parquetSchema } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/metadata.mjs";
 import { parquetReadObjects } from "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/read.mjs";
 import "../node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
@@ -238,6 +239,7 @@ async function* decodeParquetGroups(bytes, opts = {}) {
 		const rowEnd = Math.min(groupEnd, opts.rowEnd ?? Infinity);
 		if (rowStart < rowEnd) yield normalizeDecodedDates(await parquetReadObjects({
 			file,
+			compressors: { ZSTD: (input) => decompress(input) },
 			metadata: {
 				...metadata,
 				row_groups: [group],

package/dist/parquet-plan.mjs

@@ -98,7 +98,7 @@ function compileLogicalQueryPlan(plan, table = plan.dataset) {
 	const having = buildHaving(plan.metricFilters);
 	const groupByCols = [...plan.groupByDimensions.map((dimension) => dimensionToColumn(dimension, table)), ...plan.hasDate ? ["date"] : []];
 	const groupBy = groupByCols.length > 0 ? `GROUP BY ${groupByCols.join(", ")}` : "";
-	const orderBy = plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC";
+	const orderBy = `${plan.orderBy ? `ORDER BY ${plan.orderBy.column} ${plan.orderBy.dir.toUpperCase()}` : "ORDER BY clicks DESC"}${[...plan.groupByDimensions, ...plan.hasDate ? ["date"] : []].filter((d) => d !== plan.orderBy?.column).map((d) => `, ${d} ASC`).join("")}`;
 	const limit = `LIMIT ${plan.rowLimit ?? 1e3}`;
 	const offset = plan.startRow ? `OFFSET ${plan.startRow}` : "";
 	return {

package/dist/resolver/compile.mjs

@@ -28,19 +28,23 @@ function withMinimumImpressions(state, options) {
 	};
 }
 function orderByClause(state, prefix = "", columnOverride) {
+	const ties = dimensionOrder(state.dimensions, state.orderBy?.column, prefix);
 	if (state.orderBy) {
 		const safeCol = (columnOverride ?? state.orderBy.column).replace(/\W/g, "");
 		const safeDir = state.orderBy.dir.toUpperCase() === "ASC" ? "ASC" : "DESC";
-		return sql.raw(`ORDER BY ${prefix}${safeCol} ${safeDir}`);
+		return sql.raw(`ORDER BY ${prefix}${safeCol} ${safeDir}${ties}`);
 	}
-	return sql.raw(`ORDER BY ${prefix}clicks DESC`);
+	return sql.raw(`ORDER BY ${prefix}clicks DESC${ties}`);
 }
-function changeOrderClause(order) {
+function dimensionOrder(dimensions, primary, prefix = "") {
+	return [...new Set(dimensions)].filter((d) => d !== primary).map((d) => `, ${prefix}"${d.replace(/\W/g, "")}" ASC`).join("");
+}
+function changeOrderClause(order, dimensions) {
 	if (order.column !== "clicksChange") throw new Error(`[resolver] unknown comparison order column: ${JSON.stringify(order.column)}`);
 	if (order.dir !== "asc" && order.dir !== "desc") throw new Error(`[resolver] unknown comparison order direction: ${JSON.stringify(order.dir)}`);
 	return {
 		column: sql.raw("CAST(COALESCE(c.clicks, 0) AS DOUBLE) - CAST(COALESCE(p.clicks, 0) AS DOUBLE) as \"clicksChange\""),
-		orderBy: sql.raw(`ORDER BY clicksChange ${order.dir.toUpperCase()}`)
+		orderBy: sql.raw(`ORDER BY clicksChange ${order.dir.toUpperCase()}${dimensionOrder(dimensions)}`)
 	};
 }
 function limitOffsetClause(state) {
@@ -279,7 +283,7 @@ function resolveComparisonSQL(current, previous, options, comparisonFilter, comp
 	if (groupByExprs.length > 0) previousCte = sql`${previousCte} GROUP BY ${joinComma(groupByExprs)}`;
 	const joinOn = groupByDims.length > 0 ? sql.raw(groupByDims.map((d) => `c.${d.replace(/\W/g, "")} = p.${d.replace(/\W/g, "")}`).join(" AND ")) : sql.raw("1=1");
 	const filterClause = comparisonFilter ? COMPARISON_FILTER_SQL[comparisonFilter] : sql.raw("");
-	const changeOrder = comparisonOrderBy ? changeOrderClause(comparisonOrderBy) : void 0;
+	const changeOrder = comparisonOrderBy ? changeOrderClause(comparisonOrderBy, groupByDims) : void 0;
 	const orderSql = changeOrder?.orderBy ?? orderByClause(current, "");
 	const limitSql = limitOffsetClause(current);
 	const outerCurrentCols = [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "5.9.0",
+  "version": "6.0.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^5.9.0",
-    "@gscdump/lakehouse": "^5.9.0",
+    "@gscdump/contracts": "^6.0.0",
+    "@gscdump/lakehouse": "^6.0.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^5.9.0",
+    "gscdump": "^6.0.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },
@@ -202,6 +202,7 @@
     "@types/node": "^26.6.2",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
+    "fzstd": "^0.1.1",
     "hyparquet": "^1.31.1",
     "hyparquet-writer": "^0.16.10",
     "icebird": "^0.8.30",