@gscdump/engine 3.7.0 → 4.0.0

package/dist/entities/inspection.mjs

@@ -295,6 +295,15 @@ function createInspectionStore(opts) {
 				})).flat()
 			};
 		},
+		async listHistoryMonths(ctx) {
+			const prefix = inspectionHistoryPrefix(ctx, "");
+			const months = /* @__PURE__ */ new Set();
+			for (const key of await ds.list(prefix)) {
+				const month = key.slice(prefix.length).split("/")[0];
+				if (month) months.add(month);
+			}
+			return [...months].sort();
+		},
 		async materialize(ctx, rowIter) {
 			const rows = Array.from(rowIter);
 			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
@@ -310,6 +319,14 @@ function createInspectionStore(opts) {
 				bytes: bytes.byteLength
 			};
 		},
+		async loadMaterialized(ctx) {
+			const bytes = await readOptional(ds, inspectionParquetKey(ctx));
+			if (!bytes) return void 0;
+			return (await decodeParquetToRows(bytes)).map((row) => ({
+				...row,
+				scheduleNextAt: row.scheduleNextAt == null ? null : Number(row.scheduleNextAt)
+			}));
+		},
 		async appendInspectionEvents(ctx, rows, options) {
 			if (rows.length === 0) return {
 				keys: [],

package/dist/entities/sitemap-list.d.mts

@@ -0,0 +1,37 @@
+import { DataSource } from "../storage.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+export interface SitemapListContent {
+  type: string;
+  submitted: number;
+  /** Google deprecated this count. It is null when the API omits it. */
+  indexed: number | null;
+}
+export interface SitemapListEntry {
+  path: string;
+  type: string | null;
+  isPending: boolean;
+  isSitemapsIndex: boolean;
+  lastSubmitted: string | null;
+  lastDownloaded: string | null;
+  warnings: number;
+  errors: number;
+  contents: SitemapListContent[];
+}
+export interface SitemapListDoc {
+  version: 1;
+  /** ISO-8601 time of the Search Console fetch. */
+  fetchedAt: string;
+  sitemaps: SitemapListEntry[];
+}
+export interface SitemapListStore {
+  /** The last saved list, or `undefined` when none was saved. A real read failure propagates. */
+  load: (ctx: TenantCtx) => Promise<SitemapListDoc | undefined>;
+  save: (ctx: TenantCtx, doc: SitemapListDoc) => Promise<{
+    key: string;
+    bytes: number;
+  }>;
+}
+export interface CreateSitemapListStoreOptions {
+  dataSource: DataSource;
+}
+export declare function createSitemapListStore(opts: CreateSitemapListStoreOptions): SitemapListStore;

package/dist/entities/sitemap-list.mjs

@@ -0,0 +1,23 @@
+import { sitemapListKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+function createSitemapListStore(opts) {
+	const ds = opts.dataSource;
+	return {
+		async load(ctx) {
+			const bytes = await readOptional(ds, sitemapListKey(ctx));
+			if (bytes === void 0) return void 0;
+			return JSON.parse(new TextDecoder().decode(bytes));
+		},
+		async save(ctx, doc) {
+			const key = sitemapListKey(ctx);
+			const bytes = encodeJsonBigintSafe(doc);
+			await ds.write(key, bytes);
+			return {
+				key,
+				bytes: bytes.byteLength
+			};
+		}
+	};
+}
+export { createSitemapListStore };

package/dist/entities.d.mts

@@ -1,9 +1,10 @@
 import { CreateEmptyTypesStoreOptions, EmptyTypesDoc, EmptyTypesStore, createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { CreateIndexingMetadataStoreOptions, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
 import { CreateInspectionStoreOptions, INSPECTION_HISTORY_MAX_BYTES, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, createInspectionStore } from "./entities/inspection.mjs";
-import { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
 import { CompleteSitemapGeneration, CompleteSitemapTraversal, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, FinalizeSitemapGenerationResult, IterateSitemapGenerationUrlsOptions, IterateSitemapGenerationUrlsResult, ListSitemapGenerationUrlsOptions, ListSitemapGenerationUrlsResult, ParsedUrl, QuerySitemapGenerationMembershipResult, RecordSitemapLegacyImportResult, SitemapGenerationFeedManifest, SitemapGenerationReadResult, SitemapLegacyImportEvidence, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapSiteGenerationManifest, SitemapStagedFeed, SitemapStore, SitemapUrlRecord, StageSitemapGenerationFeedResult } from "./entities/sitemap-shared.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
+import { CreateSitemapListStoreOptions, SitemapListContent, SitemapListDoc, SitemapListEntry, SitemapListStore, createSitemapListStore } from "./entities/sitemap-list.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { type CompleteSitemapGeneration, type CompleteSitemapTraversal, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type EmptyTypesDoc, type EmptyTypesStore, type FinalizeSitemapGenerationResult, INSPECTION_HISTORY_MAX_BYTES, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, type IterateSitemapGenerationUrlsOptions, type IterateSitemapGenerationUrlsResult, type ListSitemapGenerationUrlsOptions, type ListSitemapGenerationUrlsResult, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type QuerySitemapGenerationMembershipResult, type RecordSitemapLegacyImportResult, SitemapFeedIdentityResult, type SitemapGenerationFeedManifest, SitemapGenerationKey, type SitemapGenerationReadResult, type SitemapLegacyImportEvidence, type SitemapMembershipEvent, type SitemapMutation, type SitemapReadStore, type SitemapSiteGenerationManifest, type SitemapStagedFeed, type SitemapStore, type SitemapUrlRecord, type StageSitemapGenerationFeedResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { type CompleteSitemapGeneration, type CompleteSitemapTraversal, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapListStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type EmptyTypesDoc, type EmptyTypesStore, type FinalizeSitemapGenerationResult, INSPECTION_HISTORY_MAX_BYTES, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, type IterateSitemapGenerationUrlsOptions, type IterateSitemapGenerationUrlsResult, type ListSitemapGenerationUrlsOptions, type ListSitemapGenerationUrlsResult, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type QuerySitemapGenerationMembershipResult, type RecordSitemapLegacyImportResult, SitemapFeedIdentityResult, type SitemapGenerationFeedManifest, SitemapGenerationKey, type SitemapGenerationReadResult, type SitemapLegacyImportEvidence, type SitemapListContent, type SitemapListDoc, type SitemapListEntry, type SitemapListStore, type SitemapMembershipEvent, type SitemapMutation, type SitemapReadStore, type SitemapSiteGenerationManifest, type SitemapStagedFeed, type SitemapStore, type SitemapUrlRecord, type StageSitemapGenerationFeedResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entities.mjs

@@ -1,8 +1,9 @@
-import { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
 import { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore } from "./entities/inspection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
+import { createSitemapListStore } from "./entities/sitemap-list.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.d.mts

@@ -20,6 +20,8 @@ export declare function inspectionHistoryPrefix(ctx: TenantCtx, yearMonth: strin
 export declare function inspectionHistoryShardKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
 /** Stable, URL-safe FNV-1a hash used for entity keys. */
 export declare function hashUrl(url: string): string;
+/** Search Console's own sitemap list for the site (path, status, counts). */
+export declare function sitemapListKey(ctx: TenantCtx): string;
 export declare function sitemapUrlsPrefix(ctx: TenantCtx): string;
 export interface SitemapGenerationKey {
   id: string;

package/dist/entity-keys.mjs

@@ -45,6 +45,9 @@ function hashUrl(url) {
 	}
 	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
 }
+function sitemapListKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/sitemaps/gsc-list.json`;
+}
 function sitemapUrlsPrefix(ctx) {
 	return `${tenantEntityPrefix(ctx)}/sitemaps/urls`;
 }
@@ -141,4 +144,4 @@ function queryDimParquetKey(ctx) {
 function queryDimMetaKey(ctx) {
 	return `${queryDimPrefix(ctx)}/index.json`;
 }
-export { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/iceberg/catalog.d.mts

@@ -1,7 +1,7 @@
 import { EngineError } from "../errors.mjs";
 import { IcebergTableName, PartitionKeyEncoding } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.28_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.28_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/types/index.mjs";
 import { Result } from "gscdump/result";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
 type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];

package/dist/iceberg/catalog.mjs

@@ -2,8 +2,8 @@ import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../drizzle-schema.mjs";
 import "../schema.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_PARTITION_SPEC, ICEBERG_TABLES, icebergSchemasFor } from "./schema.mjs";
-import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.28_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
-import "../node_modules/.pnpm/icebird@0.8.28_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
+import { icebergCreateTable } from "../node_modules/.pnpm/icebird@0.8.30_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/write/write.mjs";
+import "../node_modules/.pnpm/icebird@0.8.30_patch_hash_fddf140b38d2e0cf41ed74c5b5eb5cac324d0a6ac428572f1df7a967dcb19cae/node_modules/icebird/src/index.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";

package/dist/ingest.d.mts

@@ -36,8 +36,6 @@ export declare function toSumPosition(apiPosition: number, impressions: number):
 /**
  * Map one GSC API row into `{ date, row }` for the given table, or null if
  * the row has no keys (GSC occasionally emits empty-keys placeholders).
- * Use `createRowAccumulator` for a complete slice so URL variants combine
- * before the storage engine deduplicates stored keys.
  */
 export declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?: IngestOptions): {
   date: string;
@@ -113,4 +111,15 @@ export interface RowAccumulatorOptions extends IngestOptions {
    */
   trackDateBoundary?: boolean;
 }
+/**
+ * Combine rows that share a stored key into one row. Additive metrics
+ * (`clicks`, `impressions`, `sum_position` and their per-device variants)
+ * sum. Every other column keeps the first row's value.
+ *
+ * Distinct Google URLs can share a stored key: `https://x.com/a` and
+ * `https://www.x.com/a` both store as `/a`. Each is a separate fact, so they
+ * add. Remove repeated API rows before this step; `createRowAccumulator`
+ * does that by keying on Google's own dimension tuple.
+ */
+export declare function sumByStoredKey(table: TableName, rows: Iterable<Row>): Row[];
 export declare function createRowAccumulator(options?: RowAccumulatorOptions): RowAccumulator;

package/dist/ingest.mjs

@@ -233,17 +233,24 @@ function assembleDatesRow(date, totalsRow, deviceRows, queryGrainedImpressions)
 	};
 }
 const DEFAULT_MAX_ROWS = 5e5;
-function storedRows(table, sourceRows) {
-	if (!TABLE_DIMS[table].includes("page")) return [...sourceRows.values()];
+const ADDITIVE_COLUMN_RE = /^(?:clicks|impressions|sum_position)(?:_|$)/;
+const STORED_KEY_COLUMNS = Object.fromEntries(Object.keys(TABLE_DIMS).map((table) => [table, [.../* @__PURE__ */ new Set([
+	"date",
+	"searchAppearance",
+	...TABLE_DIMS[table].map((dimension) => dimension === "page" ? "url" : dimension)
+])]]));
+function sumByStoredKey(table, rows) {
+	const keyColumns = STORED_KEY_COLUMNS[table];
 	const grouped = /* @__PURE__ */ new Map();
-	for (const row of sourceRows.values()) {
-		const key = JSON.stringify([row.searchAppearance, ...TABLE_DIMS[table].map((dimension) => row[dimension === "page" ? "url" : dimension])]);
+	for (const row of rows) {
+		const r = row;
+		const key = JSON.stringify(keyColumns.map((col) => r[col] ?? null));
 		const prior = grouped.get(key);
-		if (prior) {
-			prior.clicks = Number(prior.clicks) + Number(row.clicks);
-			prior.impressions = Number(prior.impressions) + Number(row.impressions);
-			prior.sum_position = Number(prior.sum_position) + Number(row.sum_position);
-		} else grouped.set(key, { ...row });
+		if (!prior) {
+			grouped.set(key, { ...r });
+			continue;
+		}
+		for (const col of Object.keys(r)) if (ADDITIVE_COLUMN_RE.test(col)) prior[col] = Number(prior[col] ?? 0) + Number(r[col] ?? 0);
 	}
 	return [...grouped.values()];
 }
@@ -292,7 +299,7 @@ function createRowAccumulator(options = {}) {
 		},
 		drain() {
 			const out = /* @__PURE__ */ new Map();
-			for (const [table, byDate] of buckets) out.set(table, new Map([...byDate].map(([date, sourceRows]) => [date, storedRows(table, sourceRows)])));
+			for (const [table, byDate] of buckets) out.set(table, new Map([...byDate].map(([date, sourceRows]) => [date, sumByStoredKey(table, sourceRows.values())])));
 			buckets = /* @__PURE__ */ new Map();
 			latestDate.clear();
 			total = 0;
@@ -311,7 +318,7 @@ function createRowAccumulator(options = {}) {
 						outBy = /* @__PURE__ */ new Map();
 						out.set(table, outBy);
 					}
-					outBy.set(date, storedRows(table, dateRows));
+					outBy.set(date, sumByStoredKey(table, dateRows.values()));
 					total -= dateRows.size;
 				}
 				if (outBy) for (const date of outBy.keys()) byDate.delete(date);
@@ -320,4 +327,4 @@ function createRowAccumulator(options = {}) {
 		}
 	};
 }
-export { TABLE_DIMS, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow };
+export { TABLE_DIMS, assembleDatesRow, createRowAccumulator, sumByStoredKey, toPath, toSumPosition, transformGscRow };

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/brotli.bitreader.mjs

@@ -0,0 +1,93 @@
+const BROTLI_READ_SIZE = 4096;
+const BROTLI_IBUF_SIZE = 8224;
+const BROTLI_IBUF_MASK = 2 * BROTLI_READ_SIZE - 1;
+const kBitMask = new Uint32Array([
+	0,
+	1,
+	3,
+	7,
+	15,
+	31,
+	63,
+	127,
+	255,
+	511,
+	1023,
+	2047,
+	4095,
+	8191,
+	16383,
+	32767,
+	65535,
+	131071,
+	262143,
+	524287,
+	1048575,
+	2097151,
+	4194303,
+	8388607,
+	16777215
+]);
+var BrotliBitReader = class {
+	constructor(input) {
+		this.buf_ = new Uint8Array(BROTLI_IBUF_SIZE);
+		this.input_ = input;
+		this.buf_ptr_ = 0;
+		this.val_ = 0;
+		this.pos_ = 0;
+		this.bit_pos_ = 0;
+		this.bit_end_pos_ = 0;
+		this.eos_ = 0;
+		this.reset();
+	}
+	static READ_SIZE = BROTLI_READ_SIZE;
+	static IBUF_MASK = BROTLI_IBUF_MASK;
+	reset() {
+		this.buf_ptr_ = 0;
+		this.val_ = 0;
+		this.pos_ = 0;
+		this.bit_pos_ = 0;
+		this.bit_end_pos_ = 0;
+		this.eos_ = 0;
+		this.readMoreInput();
+		for (let i = 0; i < 4; i++) {
+			this.val_ |= this.buf_[this.pos_] << 8 * i;
+			this.pos_++;
+		}
+		return this.bit_end_pos_ > 0;
+	}
+	readMoreInput() {
+		if (this.bit_end_pos_ > 256) {} else if (this.eos_) {
+			if (this.bit_pos_ > this.bit_end_pos_) throw new Error("Unexpected end of input " + this.bit_pos_ + " " + this.bit_end_pos_);
+		} else {
+			const dst = this.buf_ptr_;
+			const bytes_read = this.input_.read(this.buf_, dst, BROTLI_READ_SIZE);
+			if (bytes_read < 0) throw new Error("Unexpected end of input");
+			if (bytes_read < BROTLI_READ_SIZE) {
+				this.eos_ = 1;
+				for (let p = 0; p < 32; p++) this.buf_[dst + bytes_read + p] = 0;
+			}
+			if (dst === 0) {
+				for (let p = 0; p < 32; p++) this.buf_[8192 + p] = this.buf_[p];
+				this.buf_ptr_ = BROTLI_READ_SIZE;
+			} else this.buf_ptr_ = 0;
+			this.bit_end_pos_ += bytes_read << 3;
+		}
+	}
+	fillBitWindow() {
+		while (this.bit_pos_ >= 8) {
+			this.val_ >>>= 8;
+			this.val_ |= this.buf_[this.pos_ & BROTLI_IBUF_MASK] << 24;
+			this.pos_++;
+			this.bit_pos_ = this.bit_pos_ - 8 >>> 0;
+			this.bit_end_pos_ = this.bit_end_pos_ - 8 >>> 0;
+		}
+	}
+	readBits(n_bits) {
+		if (32 - this.bit_pos_ < n_bits) this.fillBitWindow();
+		const val = this.val_ >>> this.bit_pos_ & kBitMask[n_bits];
+		this.bit_pos_ += n_bits;
+		return val;
+	}
+};
+export { BrotliBitReader };

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.huffman.mjs

@@ -47,10 +47,12 @@ const kMaxHuffmanTableSize = new Uint16Array([
 	1048,
 	1080
 ]);
-function HuffmanCode(bits, value) {
-	this.bits = bits;
-	this.value = value;
-}
+var HuffmanCode = class {
+	constructor(bits, value) {
+		this.bits = bits;
+		this.value = value;
+	}
+};
 const kCodeLengthRepeatCode = 16;
 const MAX_LENGTH = 15;
 function getNextKey(key, len) {
@@ -249,17 +251,19 @@ function readHuffmanCodeLengths(code_length_code_lengths, num_symbols, code_leng
 	if (space !== 0) throw new Error("[ReadHuffmanCodeLengths] space = " + space);
 	for (; symbol < num_symbols; symbol++) code_lengths[symbol] = 0;
 }
-function HuffmanTreeGroup(alphabet_size, num_htrees) {
-	this.alphabet_size = alphabet_size;
-	this.num_htrees = num_htrees;
-	this.codes = new Array(num_htrees + num_htrees * kMaxHuffmanTableSize[alphabet_size + 31 >>> 5]);
-	this.htrees = new Uint32Array(num_htrees);
-}
-HuffmanTreeGroup.prototype.decode = function(br) {
-	let next = 0;
-	for (let i = 0; i < this.num_htrees; i++) {
-		this.htrees[i] = next;
-		next += readHuffmanCode(this.alphabet_size, this.codes, next, br);
+var HuffmanTreeGroup = class {
+	constructor(alphabet_size, num_htrees) {
+		this.alphabet_size = alphabet_size;
+		this.num_htrees = num_htrees;
+		this.codes = new Array(num_htrees + num_htrees * kMaxHuffmanTableSize[alphabet_size + 31 >>> 5]);
+		this.htrees = new Uint32Array(num_htrees);
+	}
+	decode(br) {
+		let next = 0;
+		for (let i = 0; i < this.num_htrees; i++) {
+			this.htrees[i] = next;
+			next += readHuffmanCode(this.alphabet_size, this.codes, next, br);
+		}
 	}
 };
 export { HuffmanCode, HuffmanTreeGroup, readHuffmanCode, readSymbol };

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.mjs

@@ -113,7 +113,6 @@ function decompressBrotli(input, outputLength) {
 function brotli(input, output) {
 	let pos = 0;
 	let input_end = 0;
-	let window_bits = 0;
 	let max_distance = 0;
 	const dist_rb = [
 		16,
@@ -122,8 +121,6 @@ function brotli(input, output) {
 		4
 	];
 	let dist_rb_idx = 0;
-	let prev_byte1 = 0;
-	let prev_byte2 = 0;
 	const hgroup = [
 		new HuffmanTreeGroup(0, 0),
 		new HuffmanTreeGroup(0, 0),
@@ -131,7 +128,7 @@ function brotli(input, output) {
 	];
 	const kRingBufferWriteAheadSlack = 128 + BrotliBitReader.READ_SIZE;
 	const br = new BrotliBitReader(input);
-	window_bits = decodeWindowBits(br);
+	const window_bits = decodeWindowBits(br);
 	const max_backward_distance = (1 << window_bits) - 16;
 	const ringbuffer_size = 1 << window_bits;
 	const ringbuffer_mask = ringbuffer_size - 1;
@@ -144,7 +141,6 @@ function brotli(input, output) {
 		block_len_trees[x] = new HuffmanCode(0, 0);
 	}
 	while (!input_end) {
-		let meta_block_remaining_len = 0;
 		const block_length = [
 			1 << 28,
 			1 << 28,
@@ -165,14 +161,13 @@ function brotli(input, output) {
 			1
 		];
 		const block_type_rb_index = [0];
-		let context_offset = 0;
 		for (let i = 0; i < 3; i++) {
 			hgroup[i].codes = [];
 			hgroup[i].htrees = /* @__PURE__ */ new Uint32Array();
 		}
 		br.readMoreInput();
 		const _out = decodeMetaBlockLength(br);
-		meta_block_remaining_len = _out.meta_block_length;
+		let meta_block_remaining_len = _out.meta_block_length;
 		if (pos + meta_block_remaining_len > output.buffer.length) {
 			const tmp = new Uint8Array(pos + meta_block_remaining_len);
 			tmp.set(output.buffer);
@@ -246,15 +241,14 @@ function brotli(input, output) {
 			const copyIndex = kCopyRangeLut[range_idx] + (cmd_code & 7);
 			const copyCode = kCopyLengthPrefixCode[copyIndex];
 			const copyLength = copyCode.offset + br.readBits(copyCode.nbits);
-			prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
-			prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
+			let prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
+			let prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
 			for (let j = 0; j < insertLength; j++) {
 				br.readMoreInput();
 				if (block_length[0] === 0) {
 					decodeBlockType(num_block_types[0], block_type_trees, 0, block_type, block_type_rb, block_type_rb_index, br);
 					block_length[0] = readBlockLength(block_len_trees, 0, br);
-					context_offset = block_type[0] << kLiteralContextBits;
-					context_map_slice = context_offset;
+					context_map_slice = block_type[0] << kLiteralContextBits;
 					context_mode = context_modes[block_type[0]];
 					context_lookup_offset1 = lookupOffsets[context_mode];
 					context_lookup_offset2 = lookupOffsets[context_mode + 1];
@@ -326,8 +320,6 @@ function brotli(input, output) {
 					meta_block_remaining_len--;
 				}
 			}
-			prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask];
-			prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask];
 		}
 		pos &= 1073741823;
 	}

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/brotli.streams.mjs

@@ -0,0 +1,25 @@
+var BrotliInput = class {
+	constructor(buffer) {
+		this.buffer = buffer;
+		this.pos = 0;
+	}
+	read(buf, i, count) {
+		if (this.pos + count > this.buffer.length) count = this.buffer.length - this.pos;
+		for (let p = 0; p < count; p++) buf[i + p] = this.buffer[this.pos + p];
+		this.pos += count;
+		return count;
+	}
+};
+var BrotliOutput = class {
+	constructor(buf) {
+		this.buffer = buf;
+		this.pos = 0;
+	}
+	write(buf, count) {
+		if (this.pos + count > this.buffer.length) throw new Error("brotli output buffer is not large enough");
+		this.buffer.set(buf.subarray(0, count), this.pos);
+		this.pos += count;
+		return count;
+	}
+};
+export { BrotliInput, BrotliOutput };

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/brotli.transform.mjs

@@ -19,13 +19,15 @@ const kOmitFirst5 = 16;
 const kOmitFirst6 = 17;
 const kOmitFirst7 = 18;
 const kOmitFirst9 = 20;
-function Transform(prefix, transform, suffix) {
-	this.prefix = new Uint8Array(prefix.length);
-	this.transform = transform;
-	this.suffix = new Uint8Array(suffix.length);
-	for (let i = 0; i < prefix.length; i++) this.prefix[i] = prefix.charCodeAt(i);
-	for (let i = 0; i < suffix.length; i++) this.suffix[i] = suffix.charCodeAt(i);
-}
+var Transform = class {
+	constructor(prefix, transform, suffix) {
+		this.prefix = new Uint8Array(prefix.length);
+		this.transform = transform;
+		this.suffix = new Uint8Array(suffix.length);
+		for (let i = 0; i < prefix.length; i++) this.prefix[i] = prefix.charCodeAt(i);
+		for (let i = 0; i < suffix.length; i++) this.suffix[i] = suffix.charCodeAt(i);
+	}
+};
 const kTransforms = [
 	new Transform("", kIdentity, ""),
 	new Transform("", kIdentity, " "),

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -125,8 +125,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 		ensureSize(outputIndex + 131072);
 		const lms = (1 << lengthBits) - 1;
 		const dms = (1 << distBits) - 1;
-		let lpos = pos;
-		for (;; lpos = pos) {
+		for (;;) {
 			const code = lengthMap[bits16(input, pos) & lms];
 			const sym = code >> 4;
 			pos += code & 15;
@@ -134,7 +133,6 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 			if (!code) throw new Error("invalid length/literal");
 			if (sym < 256) out[outputIndex++] = sym;
 			else if (sym === 256) {
-				lpos = pos;
 				lengthMap = void 0;
 				break;
 			} else {
@@ -163,7 +161,6 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
 			}
 		}
-		pos = lpos;
 		if (lengthMap) final = 1;
 	} while (!final);
 	if (outputIndex < out.length) {

package/dist/node_modules/.pnpm/{hyparquet-compressors@1.1.1 → hyparquet-compressors@1.1.2}/node_modules/hyparquet-compressors/src/lz4.mjs

@@ -40,9 +40,10 @@ function lz4basic(input, output, outputIndex) {
 		}
 		const offset = input[i++] | input[i++] << 8;
 		if (!offset || offset > len) throw new Error(`lz4 offset out of range ${offset}`);
-		let matchLength = (token & 15) + 4;
+		let matchLength = token & 15;
 		let byte = matchLength + 240;
 		while (byte === 255) matchLength += byte = input[i++];
+		matchLength += 4;
 		let pos = len - offset;
 		const end = len + matchLength;
 		while (len < end) output[len++] = output[pos++];

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/bloom.mjs

@@ -1,4 +1,4 @@
-import { hashParquetValue } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/bloom.mjs";
+import { hashParquetValue } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/bloom.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
 const SALT = new Uint32Array([
 	1203114875,

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/column.mjs

@@ -39,7 +39,7 @@ function writeColumn({ writer, column, pageData }) {
 		encoding = userEncoding && userEncoding !== "RLE_DICTIONARY" ? userEncoding : type === "BOOLEAN" && values.length > 16 ? "RLE" : "PLAIN";
 	}
 	encodings.push(encoding);
-	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize);
+	const pageBoundaries = getPageBoundaries(writeValues, writeType, type_length, pageSize, repetitionLevels);
 	const columnIndex = column.columnIndex && pageBoundaries.length > 1 ? {
 		null_pages: [],
 		min_values: [],
@@ -141,7 +141,7 @@ function writeColumn({ writer, column, pageData }) {
 		bloomFilter
 	};
 }
-function getPageBoundaries(values, type, type_length, pageSize) {
+function getPageBoundaries(values, type, type_length, pageSize, repetitionLevels = []) {
 	if (!pageSize) return [{
 		start: 0,
 		end: values.length
@@ -152,7 +152,8 @@ function getPageBoundaries(values, type, type_length, pageSize) {
 	for (let i = 0; i < values.length; i++) {
 		const valueSize = estimateValueSize(values[i], type, type_length);
 		accumulatedBytes += valueSize;
-		if (accumulatedBytes >= pageSize && i > start) {
+		const rowStart = !repetitionLevels.length || repetitionLevels[i] === 0;
+		if (accumulatedBytes >= pageSize && i > start && rowStart) {
 			boundaries.push({
 				start,
 				end: i
@@ -200,4 +201,4 @@ function compareValues(left, right) {
 	if (left > right) return 1;
 	return 0;
 }
-export { writeColumn };
+export { getPageBoundaries, writeColumn };

package/dist/node_modules/.pnpm/{hyparquet-writer@0.16.9_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f → hyparquet-writer@0.16.10_patch_hash_d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f}/node_modules/hyparquet-writer/src/datapage.mjs

@@ -1,6 +1,6 @@
+import { Encodings, PageTypes } from "../../../../hyparquet@1.31.1/node_modules/hyparquet/src/constants.mjs";
 import { ByteWriter } from "./bytewriter.mjs";
 import { serializeTCompactProtocol } from "./thrift.mjs";
-import { Encodings, PageTypes } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/constants.mjs";
 import { deltaBinaryPack, deltaByteArray, deltaLengthByteArray } from "./delta.mjs";
 import { writeRleBitPackedHybrid } from "./encoding.mjs";
 import { writePlain } from "./plain.mjs";