@gscdump/engine 0.38.2 → 0.40.1

package/dist/index.mjs

@@ -3,7 +3,7 @@ import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, infe
 import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { enumeratePartitions } from "./_chunks/compaction.mjs";
 import { FILES_PLACEHOLDER, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
-import { engineErrorToException, engineErrors, formatEngineError, isEngineError } from "./errors.mjs";
+import { engineErrorToException, engineErrors, isEngineError } from "./errors.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
 import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
@@ -13,12 +13,6 @@ import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/source.mjs";
 import { err, ok, unwrapResult } from "gscdump/result";
-const NOOP_RESULT = {
-	flushed: 0,
-	recovered: 0,
-	failed: 0,
-	rowsWritten: 0
-};
 function scopeOf(ctx, table, date) {
 	return {
 		userId: String(ctx.userId),
@@ -28,16 +22,6 @@ function scopeOf(ctx, table, date) {
 		...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
 	};
 }
-function createNoopIngestAccumulator() {
-	return {
-		push() {
-			return false;
-		},
-		async finalize() {
-			return NOOP_RESULT;
-		}
-	};
-}
 function createIngestAccumulator(opts) {
 	const { engine, ctx, hooks, ...accOpts } = opts;
 	const acc = createRowAccumulator(accOpts);
@@ -331,4 +315,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };

package/dist/ingest.d.mts

@@ -16,8 +16,6 @@ interface GscApiRow {
   position: number;
 }
 interface IngestOptions {
-  /** @deprecated Canonical query data is built through query_dim, not fact rows. */
-  normalizeQuery?: (query: string) => string | null | undefined;
   /** Date for one-day `searchAppearance` total queries, whose keys omit date. */
   date?: string;
   /** Search appearance filter used for contextual second-step rows. */

package/dist/period/index.d.mts

@@ -1,2 +1,2 @@
-import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod } from "../_chunks/index2.mjs";
-export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod };
+import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow } from "../_chunks/index2.mjs";
+export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow };

package/dist/period/index.mjs

@@ -96,25 +96,6 @@ function resolveWindow(opts) {
 	}
 	return result;
 }
-function windowToPeriod(w) {
-	return {
-		startDate: w.start,
-		endDate: w.end
-	};
-}
-function windowToComparisonPeriod(w) {
-	if (!w.comparison) return void 0;
-	return {
-		current: {
-			startDate: w.start,
-			endDate: w.end
-		},
-		previous: {
-			startDate: w.comparison.start,
-			endDate: w.comparison.end
-		}
-	};
-}
 const DEFAULT_FILL = {
 	clicks: 0,
 	impressions: 0,
@@ -147,4 +128,4 @@ function padTimeseries(rows, options) {
 	}
 	return result;
 }
-export { comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod };
+export { comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow };

package/dist/resolver/index.d.mts

@@ -48,8 +48,6 @@ interface SqlFragmentsConfig<TableKey extends string> {
   queryDimTableRef?: () => SQL;
   /** When true, also joins query_dim on site_id (SQLite/D1 dimension table). */
   queryDimSiteScoped?: boolean;
-  /** @deprecated Canonical reads now derive from query_dim or canonical rollups. */
-  canonicalFallback?: boolean;
 }
 interface SqlFragments<TableKey extends string> {
   METRIC_NAMES: Metric[];
@@ -175,11 +173,6 @@ interface ResolveExtraFn {
 interface RunOptimizedQueryOptions {
   /** Overlay tried per extra before the live SQL; absent → today's live path. */
   resolveExtra?: ResolveExtraFn;
-  /**
-   * Deprecated compatibility flag. Canonical reads now derive from query_dim,
-   * not a nullable fact-table `query_canonical` column.
-   */
-  canonicalFallback?: boolean;
   /**
    * Versioned query dimension backing canonical reads over fact rows. Required
    * whenever a query groups or filters by `queryCanonical`; a canonical rollup
@@ -218,11 +211,6 @@ interface RunOptimizedQueryOptions {
     coversThrough?: string;
     queryDim?: CanonicalQueryDimSource;
   };
-  /**
-   * @deprecated Canonical-source misses are recorded on `source.fallbacks`;
-   * raw daily fallback is controlled by `primarySourceFallback: 'raw'`.
-   */
-  canonicalSourceFallback?: 'raw';
   /** Optional version gates for the query dimension backing `canonicalSource`. */
   canonicalRequirements?: {
     normalizerVersion?: number;

package/dist/rollups.mjs

@@ -195,26 +195,42 @@ function planRollupWindows(parts, clampRange, maxWindowDays) {
 		});
 	}
 	if (spans.length === 0) return [];
-	let rangeStartMs = Math.min(...spans.map((s) => s.startMs));
-	let rangeEndMs = Math.max(...spans.map((s) => s.endMs));
+	let rangeStartMs = Number.POSITIVE_INFINITY;
+	let rangeEndMs = Number.NEGATIVE_INFINITY;
+	let totalBytes = 0;
+	for (const span of spans) {
+		if (span.startMs < rangeStartMs) rangeStartMs = span.startMs;
+		if (span.endMs > rangeEndMs) rangeEndMs = span.endMs;
+		totalBytes += span.bytes;
+	}
 	if (clampStartMs !== void 0) rangeStartMs = Math.max(rangeStartMs, clampStartMs);
 	if (clampEndMs !== void 0) rangeEndMs = Math.min(rangeEndMs, clampEndMs);
-	const totalBytes = spans.reduce((a, s) => a + s.bytes, 0);
 	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
 	const bytesPerDay = Math.max(1, totalBytes / spanDays);
 	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
 	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
+	if (!Number.isFinite(windowDays) || !Number.isFinite(rangeStartMs) || !Number.isFinite(rangeEndMs) || rangeEndMs < rangeStartMs) return [];
+	const windowWidthMs = windowDays * MS_PER_DAY;
+	const windowCount = Math.floor((rangeEndMs - rangeStartMs) / windowWidthMs) + 1;
+	const partitionsByWindow = Array.from({ length: windowCount }, () => []);
+	for (const span of spans) {
+		const overlapStartMs = Math.max(span.startMs, rangeStartMs);
+		const overlapEndMs = Math.min(span.endMs, rangeEndMs);
+		if (overlapStartMs > overlapEndMs) continue;
+		const firstWindow = Math.max(0, Math.ceil((overlapStartMs - rangeStartMs - (windowDays - 1) * MS_PER_DAY) / windowWidthMs));
+		const lastWindow = Math.min(windowCount - 1, Math.floor((overlapEndMs - rangeStartMs) / windowWidthMs));
+		for (let index = firstWindow; index <= lastWindow; index++) partitionsByWindow[index].push(span.partition);
+	}
 	const windows = [];
-	let cursorMs = rangeStartMs;
-	while (cursorMs <= rangeEndMs) {
+	for (let index = 0; index < windowCount; index++) {
+		const cursorMs = rangeStartMs + index * windowWidthMs;
 		const windowEndMs = Math.min(cursorMs + (windowDays - 1) * MS_PER_DAY, rangeEndMs);
-		const partitions = spans.filter((s) => s.endMs >= cursorMs && s.startMs <= windowEndMs).map((s) => s.partition);
+		const partitions = partitionsByWindow[index];
 		if (partitions.length > 0) windows.push({
 			start: isoDate(cursorMs),
 			end: isoDate(windowEndMs),
 			partitions
 		});
-		cursorMs = windowEndMs + MS_PER_DAY;
 	}
 	return windows;
 }

package/dist/schema.d.mts

@@ -1,2 +1,2 @@
-import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
-export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
+export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/schema.mjs

@@ -1,2 +1,2 @@
-import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/sink-node.d.mts

@@ -17,9 +17,8 @@ interface OverwriteWriterCatalogConfig {
   catalogToken?: string;
 }
 /**
- * The JSON job wire-format shared with `scripts/iceberg-writer.py` — identical
- * to the shape `LocalIcebergSink` already emits, so one PyIceberg script backs
- * both the local sink and this writer. `op` is always `overwrite` here.
+ * The JSON job wire-format shared with `scripts/iceberg-writer.py`.
+ * `op` is always `overwrite` here.
  */
 interface OverwriteJob {
   op: 'overwrite';

package/dist/source/index.d.mts

@@ -1,3 +1,3 @@
 import { AnalysisQuerySource, AnalysisSourceKind, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
-import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, TypedQuery, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery } from "../_chunks/index.mjs";
-export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, TypedQuery, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
+import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine } from "../_chunks/index.mjs";
+export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };

package/dist/source/index.mjs

@@ -1,2 +1,2 @@
-import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery } from "../_chunks/source.mjs";
-export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
+import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine } from "../_chunks/source.mjs";
+export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.38.2",
+  "version": "0.40.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/lakehouse": "0.38.2",
-    "gscdump": "0.38.2",
-    "@gscdump/contracts": "0.38.2"
+    "@gscdump/contracts": "0.40.1",
+    "@gscdump/lakehouse": "0.40.1",
+    "gscdump": "0.40.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",