@gscdump/engine 0.40.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +4 -4
  2. package/dist/_chunks/duckdb.d.mts +2 -9
  3. package/dist/_chunks/engine.mjs +3 -7
  4. package/dist/_chunks/entities.mjs +1 -1
  5. package/dist/_chunks/index.d.mts +3 -8
  6. package/dist/_chunks/index2.d.mts +1 -2
  7. package/dist/_chunks/libs/hyparquet-compressors.mjs +1 -1
  8. package/dist/_chunks/libs/hyparquet.mjs +1 -63
  9. package/dist/_chunks/libs/icebird.d.mts +1 -81
  10. package/dist/_chunks/libs/icebird.mjs +4 -330
  11. package/dist/_chunks/parquet-plan.mjs +248 -3
  12. package/dist/_chunks/resolver.mjs +16 -2
  13. package/dist/_chunks/schema.d.mts +2 -2
  14. package/dist/_chunks/sink.d.mts +4 -5
  15. package/dist/_chunks/source.mjs +3 -3
  16. package/dist/_chunks/storage.d.mts +1 -43
  17. package/dist/_chunks/types.d.mts +8 -0
  18. package/dist/adapters/node.d.mts +1 -11
  19. package/dist/adapters/node.mjs +1 -1
  20. package/dist/analyzer/index.d.mts +1 -1
  21. package/dist/entities.d.mts +2 -10
  22. package/dist/entities.mjs +2 -2
  23. package/dist/iceberg/index.d.mts +2 -3
  24. package/dist/iceberg/index.mjs +12 -12
  25. package/dist/index.d.mts +83 -2
  26. package/dist/index.mjs +365 -6
  27. package/dist/period/index.d.mts +2 -2
  28. package/dist/period/index.mjs +1 -1
  29. package/dist/planner.mjs +1 -2
  30. package/dist/resolver/index.d.mts +2 -9
  31. package/dist/resolver/index.mjs +2 -2
  32. package/dist/rollups.d.mts +3 -20
  33. package/dist/rollups.mjs +5 -5
  34. package/dist/scope.d.mts +1 -2
  35. package/dist/scope.mjs +1 -1
  36. package/dist/source/index.d.mts +2 -2
  37. package/dist/source/index.mjs +2 -2
  38. package/dist/sql-bind.d.mts +1 -29
  39. package/dist/sql-bind.mjs +1 -1
  40. package/package.json +7 -22
  41. package/dist/_chunks/compaction.mjs +0 -251
  42. package/dist/adapters/r2-manifest.d.mts +0 -82
  43. package/dist/adapters/r2-manifest.mjs +0 -364
  44. package/dist/compaction-public.d.mts +0 -15
  45. package/dist/compaction-public.mjs +0 -5
  46. package/dist/snapshot.d.mts +0 -2
  47. package/dist/snapshot.mjs +0 -1
package/dist/index.d.mts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
2
- import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
2
+ import { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
3
3
  import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
4
+ import { SnapshotIndex } from "./_chunks/snapshot.mjs";
4
5
  import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, isEngineError } from "./_chunks/errors.mjs";
5
6
  import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
6
7
  import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
@@ -13,6 +14,86 @@ import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
13
14
  import { Result } from "gscdump/result";
14
15
  import { SearchType as SearchType$1 } from "gscdump/query";
15
16
  import { Grain as Grain$1, Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
17
+ interface R2ObjectMetadata {
18
+ etag: string;
19
+ }
20
+ interface R2ObjectBody extends R2ObjectMetadata {
21
+ text: () => Promise<string>;
22
+ }
23
+ interface R2ListResult {
24
+ objects: Array<{
25
+ key: string;
26
+ }>;
27
+ truncated: boolean;
28
+ cursor?: string;
29
+ }
30
+ interface R2ConditionalPutOptions {
31
+ /**
32
+ * Workers-binding-style precondition. `etagMatches` rejects with `null`
33
+ * return on mismatch; `etagDoesNotMatch: '*'` rejects if the key exists.
34
+ */
35
+ onlyIf?: {
36
+ etagMatches?: string;
37
+ etagDoesNotMatch?: string;
38
+ };
39
+ }
40
+ /**
41
+ * Minimal Cloudflare R2 binding shape needed for the manifest CAS loop.
42
+ * Structurally compatible with Cloudflare's `R2Bucket` Workers API.
43
+ */
44
+ interface R2ManifestBucketLike {
45
+ get: (key: string) => Promise<R2ObjectBody | null>;
46
+ put: (key: string, bytes: string | Uint8Array, options?: R2ConditionalPutOptions) => Promise<R2ObjectMetadata | null>;
47
+ list: (options?: {
48
+ prefix?: string;
49
+ cursor?: string;
50
+ limit?: number;
51
+ }) => Promise<R2ListResult>;
52
+ /**
53
+ * Bulk delete. Required by {@link ManifestStore.purgeTenant}. Cloudflare's
54
+ * `R2Bucket.delete` accepts a single key or a string[] batch; both shapes
55
+ * work here.
56
+ */
57
+ delete: (keys: string | string[]) => Promise<void>;
58
+ }
59
+ /**
60
+ * CAS lifecycle events emitted by the manifest store. Consumers wire these
61
+ * into metrics (prom-client, console.table, the contention harness) to
62
+ * measure rejection rate and latency under real R2 load.
63
+ */
64
+ type R2ManifestEvent = {
65
+ kind: 'cas-attempt';
66
+ siteId: string;
67
+ table: TableName;
68
+ attempt: number;
69
+ } | {
70
+ kind: 'cas-rejected';
71
+ siteId: string;
72
+ table: TableName;
73
+ attempt: number;
74
+ } | {
75
+ kind: 'cas-committed';
76
+ siteId: string;
77
+ table: TableName;
78
+ attempts: number;
79
+ };
80
+ interface CreateR2ManifestStoreOptions {
81
+ bucket: R2ManifestBucketLike;
82
+ /** Tenant scope. All shard keys are prefixed `u_<userId>/manifest/...`. */
83
+ userId: string;
84
+ /** Override the snapshot version-id generator. Defaults to `${ts}-${random}`. */
85
+ newSnapshotId?: () => string;
86
+ now?: () => number;
87
+ /** Maximum CAS retries before giving up. Defaults to 8. */
88
+ maxRetries?: number;
89
+ /**
90
+ * Optional telemetry hook. Fired synchronously from the CAS loop on each
91
+ * attempt, rejection, and successful commit. Must not throw; exceptions
92
+ * propagate and will fail the mutation.
93
+ */
94
+ onEvent?: (event: R2ManifestEvent) => void;
95
+ }
96
+ declare function createR2ManifestStore(opts: CreateR2ManifestStoreOptions): ManifestStore;
16
97
  declare function coerceRow(row: Row$1): Row$1;
17
98
  declare function coerceRows(rows: readonly Row$1[]): Row$1[];
18
99
  declare const MAX_DAY_BYTES: number;
@@ -208,4 +289,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
208
289
  declare const MIN_COUNTRY_IMPRESSIONS = 10;
209
290
  declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
210
291
  declare const MAX_TRACKED_URLS_PER_SITE = 200000;
211
- export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
292
+ export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, type R2ManifestBucketLike, type R2ManifestEvent, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type SnapshotIndex, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2ManifestStore, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
package/dist/index.mjs CHANGED
@@ -1,11 +1,11 @@
1
- import { coerceRow, coerceRows } from "./_chunks/coerce.mjs";
1
+ import { engineErrorToException, engineErrors, isEngineError } from "./errors.mjs";
2
2
  import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/layout.mjs";
3
+ import { matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter } from "./_chunks/manifest-store-utils.mjs";
4
+ import { coerceRow, coerceRows } from "./_chunks/coerce.mjs";
3
5
  import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
4
- import { enumeratePartitions } from "./_chunks/compaction.mjs";
5
- import { FILES_PLACEHOLDER, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
6
- import { engineErrorToException, engineErrors, isEngineError } from "./errors.mjs";
6
+ import { FILES_PLACEHOLDER, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
7
7
  import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
8
- import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
8
+ import { MAX_DAY_BYTES, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
9
9
  import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
10
10
  import "./planner.mjs";
11
11
  import { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "./_chunks/resolver.mjs";
@@ -13,6 +13,365 @@ import { rebuildDailyFromHourly } from "./rollups.mjs";
13
13
  import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
14
14
  import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/source.mjs";
15
15
  import { err, ok, unwrapResult } from "gscdump/result";
16
+ const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
17
+ const CAS_BACKOFF_BASE_MS = 5;
18
+ const CAS_BACKOFF_CAP_MS = 250;
19
+ const SHARD_IO_CONCURRENCY = 8;
20
+ const R2_DELETE_BATCH_SIZE = 1e3;
21
+ async function casBackoff(attempt) {
22
+ const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
23
+ await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
24
+ }
25
+ async function mapWithConcurrency(items, concurrency, fn) {
26
+ if (items.length === 0) return [];
27
+ const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
28
+ const results = Array.from({ length: items.length }, () => void 0);
29
+ let nextIndex = 0;
30
+ async function worker() {
31
+ while (true) {
32
+ const index = nextIndex++;
33
+ if (index >= items.length) return;
34
+ results[index] = await fn(items[index], index);
35
+ }
36
+ }
37
+ await Promise.all(Array.from({ length: workerCount }, worker));
38
+ return results;
39
+ }
40
+ function defaultSnapshotId() {
41
+ return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
42
+ }
43
+ function shardPrefix(userId, siteId, table) {
44
+ return `u_${userId}/manifest/${siteId}/${table}/`;
45
+ }
46
+ function headKey(userId, siteId, table) {
47
+ return `${shardPrefix(userId, siteId, table)}HEAD`;
48
+ }
49
+ function snapshotKey(userId, siteId, table, snapshotId) {
50
+ return `${shardPrefix(userId, siteId, table)}v${snapshotId}.json`;
51
+ }
52
+ function emptySnapshot() {
53
+ return {
54
+ version: 1,
55
+ entries: [],
56
+ watermarks: [],
57
+ syncStates: []
58
+ };
59
+ }
60
+ function shardScopesFromEntries(entries) {
61
+ const out = /* @__PURE__ */ new Set();
62
+ for (const e of entries) {
63
+ if (e.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId; cross-site entries are unshardable");
64
+ out.add(`${e.siteId}\0${e.table}`);
65
+ }
66
+ return out;
67
+ }
68
+ function createR2ManifestStore(opts) {
69
+ const { bucket, userId } = opts;
70
+ const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
71
+ const now = opts.now ?? (() => Date.now());
72
+ const maxRetries = opts.maxRetries ?? 16;
73
+ const onEvent = opts.onEvent;
74
+ async function readShard(siteId, table) {
75
+ const head = await bucket.get(headKey(userId, siteId, table));
76
+ if (!head) return {
77
+ snapshot: emptySnapshot(),
78
+ headEtag: void 0
79
+ };
80
+ const snapshotId = (await head.text()).trim();
81
+ if (!snapshotId) return {
82
+ snapshot: emptySnapshot(),
83
+ headEtag: head.etag
84
+ };
85
+ const snap = await bucket.get(snapshotKey(userId, siteId, table, snapshotId));
86
+ if (!snap) return {
87
+ snapshot: emptySnapshot(),
88
+ headEtag: head.etag
89
+ };
90
+ const parsed = JSON.parse(await snap.text());
91
+ if (parsed.version !== 1) throw new Error(`unsupported manifest snapshot version: ${parsed.version}`);
92
+ return {
93
+ snapshot: parsed,
94
+ headEtag: head.etag
95
+ };
96
+ }
97
+ async function writeShardResult(siteId, table, snapshot, headEtag, attempt) {
98
+ const id = newSnapshotId();
99
+ const snapKey = snapshotKey(userId, siteId, table, id);
100
+ await bucket.put(snapKey, JSON.stringify(snapshot));
101
+ const conditional = headEtag ? { onlyIf: { etagMatches: headEtag } } : { onlyIf: { etagDoesNotMatch: "*" } };
102
+ return await bucket.put(headKey(userId, siteId, table), id, conditional) !== null ? ok(void 0) : err(engineErrors.manifestCasRoundLost(siteId, table, attempt));
103
+ }
104
+ async function mutateShardResult(siteId, table, mutate) {
105
+ let attempt = 0;
106
+ while (attempt < maxRetries) {
107
+ onEvent?.({
108
+ kind: "cas-attempt",
109
+ siteId,
110
+ table,
111
+ attempt
112
+ });
113
+ const { snapshot, headEtag } = await readShard(siteId, table);
114
+ await mutate(snapshot);
115
+ const round = await writeShardResult(siteId, table, snapshot, headEtag, attempt);
116
+ if (round.ok) {
117
+ onEvent?.({
118
+ kind: "cas-committed",
119
+ siteId,
120
+ table,
121
+ attempts: attempt + 1
122
+ });
123
+ return round;
124
+ }
125
+ onEvent?.({
126
+ kind: "cas-rejected",
127
+ siteId,
128
+ table,
129
+ attempt
130
+ });
131
+ attempt++;
132
+ if (attempt < maxRetries) await casBackoff(attempt);
133
+ }
134
+ return err(engineErrors.manifestCasExhausted(siteId, table, maxRetries));
135
+ }
136
+ async function mutateShard(siteId, table, mutate) {
137
+ return unwrapResult(await mutateShardResult(siteId, table, mutate), engineErrorToException);
138
+ }
139
+ async function listShards(siteId) {
140
+ const shards = [];
141
+ let cursor;
142
+ const prefix = siteId === void 0 ? `u_${userId}/manifest/` : `u_${userId}/manifest/${siteId}/`;
143
+ do {
144
+ const res = await bucket.list({
145
+ prefix,
146
+ cursor,
147
+ limit: 1e3
148
+ });
149
+ for (const obj of res.objects) {
150
+ const m = SHARD_RE.exec(obj.key);
151
+ if (m?.groups) shards.push({
152
+ siteId: m.groups.siteId,
153
+ table: m.groups.table
154
+ });
155
+ }
156
+ cursor = res.truncated ? res.cursor : void 0;
157
+ } while (cursor);
158
+ return shards;
159
+ }
160
+ async function shardsForFilter(filter) {
161
+ if (filter.siteId !== void 0 && filter.table !== void 0) return [{
162
+ siteId: filter.siteId,
163
+ table: filter.table
164
+ }];
165
+ return (await listShards(filter.siteId)).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
166
+ }
167
+ function assertScopedUser(got, op) {
168
+ if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
169
+ }
170
+ async function readEntriesAcrossShards(filter, includeRetired) {
171
+ assertScopedUser(filter.userId, includeRetired ? "listAll" : "listLive");
172
+ return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
173
+ const { snapshot } = await readShard(siteId, table);
174
+ const entries = [];
175
+ for (const entry of snapshot.entries) {
176
+ if (!includeRetired && entry.retiredAt !== void 0) continue;
177
+ if (matchesManifestEntryFilter(entry, filter, { ignoreUserId: true })) entries.push(entry);
178
+ }
179
+ return entries;
180
+ })).flat();
181
+ }
182
+ function groupBySiteTable(entries) {
183
+ const out = /* @__PURE__ */ new Map();
184
+ for (const e of entries) {
185
+ const key = `${e.siteId}\0${e.table}`;
186
+ if (!out.has(key)) out.set(key, []);
187
+ out.get(key).push(e);
188
+ }
189
+ return out;
190
+ }
191
+ async function registerVersionsImpl(newEntries, superseding) {
192
+ if (newEntries.length === 0 && (!superseding || superseding.length === 0)) return;
193
+ const supersededAt = newEntries[0]?.createdAt ?? now();
194
+ const byShard = /* @__PURE__ */ new Map();
195
+ function bucket(entry, kind) {
196
+ assertScopedUser(entry.userId, "registerVersions");
197
+ if (entry.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId");
198
+ const key = `${entry.siteId}\0${entry.table}`;
199
+ let bag = byShard.get(key);
200
+ if (!bag) {
201
+ bag = {
202
+ newEntries: [],
203
+ superseding: []
204
+ };
205
+ byShard.set(key, bag);
206
+ }
207
+ if (kind === "new") bag.newEntries.push(entry);
208
+ else bag.superseding.push(entry);
209
+ }
210
+ for (const e of newEntries) bucket(e, "new");
211
+ if (superseding) for (const e of superseding) bucket(e, "super");
212
+ await mapWithConcurrency([...byShard], SHARD_IO_CONCURRENCY, async ([shardKey, { newEntries: news, superseding: supers }]) => {
213
+ const [siteId, table] = shardKey.split("\0");
214
+ await mutateShard(siteId, table, (snap) => {
215
+ const byObjectKey = new Map(snap.entries.map((e) => [e.objectKey, e]));
216
+ for (const s of supers) {
217
+ const existing = byObjectKey.get(s.objectKey);
218
+ if (existing && existing.retiredAt === void 0) byObjectKey.set(s.objectKey, {
219
+ ...existing,
220
+ retiredAt: supersededAt
221
+ });
222
+ }
223
+ for (const n of news) byObjectKey.set(n.objectKey, n);
224
+ snap.entries = Array.from(byObjectKey.values());
225
+ });
226
+ });
227
+ }
228
+ return {
229
+ async listLive(filter) {
230
+ return readEntriesAcrossShards(filter, false);
231
+ },
232
+ async listAll(filter) {
233
+ return readEntriesAcrossShards(filter, true);
234
+ },
235
+ async registerVersion(entry, superseding) {
236
+ return registerVersionsImpl([entry], superseding);
237
+ },
238
+ async registerVersions(entries, superseding) {
239
+ shardScopesFromEntries(entries);
240
+ return registerVersionsImpl(entries, superseding);
241
+ },
242
+ async listRetired(olderThan) {
243
+ return (await mapWithConcurrency(await listShards(), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
244
+ const { snapshot } = await readShard(siteId, table);
245
+ const retired = [];
246
+ for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) retired.push(e);
247
+ return retired;
248
+ })).flat();
249
+ },
250
+ async delete(toDelete) {
251
+ await mapWithConcurrency([...groupBySiteTable(toDelete)], SHARD_IO_CONCURRENCY, async ([shardKey, entries]) => {
252
+ const [siteId, table] = shardKey.split("\0");
253
+ await mutateShard(siteId, table, (snap) => {
254
+ const drop = new Set(entries.map((e) => e.objectKey));
255
+ snap.entries = snap.entries.filter((e) => !drop.has(e.objectKey));
256
+ });
257
+ });
258
+ },
259
+ async getWatermarks(filter) {
260
+ assertScopedUser(filter.userId, "getWatermarks");
261
+ return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
262
+ const { snapshot } = await readShard(siteId, table);
263
+ const watermarks = [];
264
+ for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter, { ignoreUserId: true })) watermarks.push(w);
265
+ return watermarks;
266
+ })).flat();
267
+ },
268
+ async bumpWatermark(scope, date, at) {
269
+ assertScopedUser(scope.userId, "bumpWatermark");
270
+ if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
271
+ const ts = at ?? now();
272
+ const scopeSearchType = inferSearchType(scope);
273
+ await mutateShard(scope.siteId, scope.table, (snap) => {
274
+ const idx = snap.watermarks.findIndex((w) => w.userId === userId && w.siteId === scope.siteId && w.table === scope.table && inferSearchType(w) === scopeSearchType);
275
+ if (idx === -1) {
276
+ snap.watermarks.push({
277
+ userId,
278
+ siteId: scope.siteId,
279
+ table: scope.table,
280
+ ...scope.searchType !== void 0 ? { searchType: scope.searchType } : {},
281
+ newestDateSynced: date,
282
+ oldestDateSynced: date,
283
+ lastSyncAt: ts
284
+ });
285
+ return;
286
+ }
287
+ const w = snap.watermarks[idx];
288
+ const newest = date > w.newestDateSynced ? date : w.newestDateSynced;
289
+ const oldest = date < w.oldestDateSynced ? date : w.oldestDateSynced;
290
+ const lastSyncAt = ts > w.lastSyncAt ? ts : w.lastSyncAt;
291
+ snap.watermarks[idx] = {
292
+ ...w,
293
+ newestDateSynced: newest,
294
+ oldestDateSynced: oldest,
295
+ lastSyncAt
296
+ };
297
+ });
298
+ },
299
+ async getSyncStates(filter) {
300
+ assertScopedUser(filter.userId, "getSyncStates");
301
+ return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
302
+ const { snapshot } = await readShard(siteId, table);
303
+ const states = [];
304
+ for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter, { ignoreUserId: true })) states.push(s);
305
+ return states;
306
+ })).flat();
307
+ },
308
+ async setSyncState(scope, state, detail) {
309
+ assertScopedUser(scope.userId, "setSyncState");
310
+ if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
311
+ const at = detail?.at ?? now();
312
+ const scopeSearchType = inferSearchType(scope);
313
+ await mutateShard(scope.siteId, scope.table, (snap) => {
314
+ const idx = snap.syncStates.findIndex((s) => s.userId === userId && s.siteId === scope.siteId && s.table === scope.table && s.date === scope.date && inferSearchType(s) === scopeSearchType);
315
+ if (idx === -1) {
316
+ snap.syncStates.push({
317
+ userId,
318
+ siteId: scope.siteId,
319
+ table: scope.table,
320
+ date: scope.date,
321
+ state,
322
+ updatedAt: at,
323
+ attempts: 1,
324
+ error: detail?.error,
325
+ ...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
326
+ });
327
+ return;
328
+ }
329
+ const prev = snap.syncStates[idx];
330
+ const attempts = state === "inflight" && prev.state !== "inflight" ? prev.attempts + 1 : prev.attempts;
331
+ const error = state === "done" ? void 0 : detail?.error ?? prev.error;
332
+ snap.syncStates[idx] = {
333
+ ...prev,
334
+ state,
335
+ updatedAt: at,
336
+ attempts,
337
+ error
338
+ };
339
+ });
340
+ },
341
+ async withLock(_scope, fn) {
342
+ return fn();
343
+ },
344
+ async purgeTenant(filter) {
345
+ if (filter.userId !== userId) throw new Error(`purgeTenant: store is scoped to userId=${userId}, got ${filter.userId}`);
346
+ const purged = await mapWithConcurrency(await shardsForFilter({ siteId: filter.siteId }), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
347
+ const { snapshot } = await readShard(siteId, table);
348
+ const prefix = shardPrefix(userId, siteId, table);
349
+ const keys = [];
350
+ let cursor;
351
+ do {
352
+ const res = await bucket.list({
353
+ prefix,
354
+ cursor,
355
+ limit: 1e3
356
+ });
357
+ for (const obj of res.objects) keys.push(obj.key);
358
+ cursor = res.truncated ? res.cursor : void 0;
359
+ } while (cursor);
360
+ for (let offset = 0; offset < keys.length; offset += R2_DELETE_BATCH_SIZE) await bucket.delete(keys.slice(offset, offset + R2_DELETE_BATCH_SIZE));
361
+ return {
362
+ entriesRemoved: snapshot.entries.length,
363
+ watermarksRemoved: snapshot.watermarks.length,
364
+ syncStatesRemoved: snapshot.syncStates.length
365
+ };
366
+ });
367
+ return {
368
+ entriesRemoved: purged.reduce((sum, result) => sum + result.entriesRemoved, 0),
369
+ watermarksRemoved: purged.reduce((sum, result) => sum + result.watermarksRemoved, 0),
370
+ syncStatesRemoved: purged.reduce((sum, result) => sum + result.syncStatesRemoved, 0)
371
+ };
372
+ }
373
+ };
374
+ }
16
375
  function scopeOf(ctx, table, date) {
17
376
  return {
18
377
  userId: String(ctx.userId),
@@ -315,4 +674,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
315
674
  const MIN_COUNTRY_IMPRESSIONS = 10;
316
675
  const MAX_SITEMAP_URLS_PER_SITE = 5e4;
317
676
  const MAX_TRACKED_URLS_PER_SITE = 2e5;
318
- export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
677
+ export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2ManifestStore, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
@@ -1,2 +1,2 @@
1
- import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow } from "../_chunks/index2.mjs";
2
- export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow };
1
+ import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow } from "../_chunks/index2.mjs";
2
+ export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
@@ -128,4 +128,4 @@ function padTimeseries(rows, options) {
128
128
  }
129
129
  return result;
130
130
  }
131
- export { comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow };
131
+ export { comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
package/dist/planner.mjs CHANGED
@@ -1,3 +1,2 @@
1
- import { enumeratePartitions } from "./_chunks/compaction.mjs";
2
- import { FILES_PLACEHOLDER, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
1
+ import { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
3
2
  export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };
@@ -1,7 +1,7 @@
1
1
  import { SearchType as SearchType$1, TableName as TableName$1 } from "../_chunks/storage.mjs";
2
2
  import { ComparisonFilter, ExtraQuery, ResolvedComparisonSQL, ResolvedSQL, ResolvedSQLOptimized, ResolverAdapter, ResolverOptions } from "../_chunks/types.mjs";
3
3
  import { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "../_chunks/pg-adapter.mjs";
4
- import { LogicalDataset, LogicalDataset as LogicalDataset$1, LogicalQueryPlan, PlannerCapabilities, UnresolvableDatasetError, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
4
+ import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities, UnresolvableDatasetError, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
5
5
  import { SQL } from "drizzle-orm";
6
6
  import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
7
7
  import { Grain, TableName } from "@gscdump/contracts";
@@ -81,13 +81,6 @@ interface CreateResolverAdapterConfig<TableKey extends string> extends SqlFragme
81
81
  capabilities: PlannerCapabilities;
82
82
  }
83
83
  declare function createResolverAdapter<TableKey extends string>(config: CreateResolverAdapterConfig<TableKey>): ResolverAdapter<TableKey>;
84
- /**
85
- * True when `plan` can be served from the canonical-grained rollup instead of
86
- * the raw `queries` fact partitions. Conservative: anything that would read a
87
- * dropped column or the raw row grain disqualifies the query. Callers decide
88
- * whether that miss is a typed raw fallback or a hard coverage error.
89
- */
90
- declare function planCoveredByCanonicalRollup(plan: LogicalQueryPlan): boolean;
91
84
  /** State-level convenience: build the plan then gate. */
92
85
  declare function canonicalRollupCovers(state: BuilderState, capabilities: PlannerCapabilities): boolean;
93
86
  declare function resolveToSQLOptimized<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ResolvedSQLOptimized;
@@ -324,4 +317,4 @@ interface AssertSchemaInSyncOptions {
324
317
  mode: 'exact' | 'superset';
325
318
  }
326
319
  declare function assertSchemaInSync(options: AssertSchemaInSyncOptions): void;
327
- export { type AssertSchemaInSyncOptions, type CanonicalQueryDimSource, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type PrimaryColumnarSource, QuerySourceCoverageError, type QuerySourceDecision, type QuerySourceFallback, type QuerySourceFallbackKind, type QuerySourceKind, type R2SqlResolverAdapterOptions, type ResolveExtraFn, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverAdapterOptions, type ResolverOptions, type RollupRowsReader, type RunOptimizedQueryOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
320
+ export { type AssertSchemaInSyncOptions, type CanonicalQueryDimSource, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type PrimaryColumnarSource, QuerySourceCoverageError, type QuerySourceDecision, type QuerySourceFallback, type QuerySourceFallbackKind, type QuerySourceKind, type R2SqlResolverAdapterOptions, type ResolveExtraFn, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverAdapterOptions, type ResolverOptions, type RollupRowsReader, type RunOptimizedQueryOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
@@ -1,2 +1,2 @@
1
- import { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
2
- export { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
1
+ import { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
2
+ export { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
@@ -1,5 +1,5 @@
1
1
  import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "./_chunks/storage.mjs";
2
- import { ColumnDef as ColumnDef$1 } from "./_chunks/schema.mjs";
2
+ import { ColumnDef } from "./_chunks/schema.mjs";
3
3
  import { EngineError } from "./_chunks/errors.mjs";
4
4
  import "./_chunks/contracts.mjs";
5
5
  import { SearchType } from "gscdump/query";
@@ -71,7 +71,7 @@ interface RollupDef {
71
71
  * Types map the same way as the fact-table encoder: VARCHAR / DATE go
72
72
  * through BYTE_ARRAY/UTF8; BIGINT → INT64; INTEGER → INT32; DOUBLE → DOUBLE.
73
73
  */
74
- parquetColumns?: readonly ColumnDef$1[];
74
+ parquetColumns?: readonly ColumnDef[];
75
75
  /** Sort-key column names for parquet row-group stats. Optional. */
76
76
  parquetSortKey?: readonly string[];
77
77
  /**
@@ -231,16 +231,6 @@ declare const WINDOW_BYTE_BUDGET: number;
231
231
  declare const ROLLUP_PAGE_ROWS = 50000;
232
232
  declare const ROLLUP_PAGE_ROWS_WIDE = 20000;
233
233
  declare const ROLLUP_PAGE_ROWS_DAILY = 70000;
234
- declare const DAILY_MAX_WINDOW_DAYS = 7;
235
- /**
236
- * UTC day-aligned [startMs, endMs] span a partition covers. Returns null for
237
- * `hourly/` partitions and anything unrecognised — those are excluded from
238
- * windowed planning.
239
- */
240
- declare function partitionDaySpan(partition: string): {
241
- startMs: number;
242
- endMs: number;
243
- } | null;
244
234
  /**
245
235
  * Plan byte-bounded windows over a partition set. Each window names the
246
236
  * partitions whose span intersects it; a coarse tier file can land in two
@@ -257,11 +247,6 @@ declare function planRollupWindows(parts: Array<{
257
247
  end: string;
258
248
  partitions: string[];
259
249
  }>;
260
- /** Partition strings whose span intersects the inclusive [start, end] date range. */
261
- declare function partitionsInRange(parts: Array<{
262
- partition: string;
263
- bytes: number;
264
- }>, start: string, end: string): string[];
265
250
  /**
266
251
  * Run a full-history aggregation in byte-bounded windows and concat the rows.
267
252
  * Each window's SQL MUST date-filter to `[w.start, w.end]` (see `sqlFor`) so a
@@ -322,8 +307,6 @@ declare const topPages28dRollup: RollupDef;
322
307
  * spinning up DuckDB-WASM.
323
308
  */
324
309
  declare const topCountries28dRollup: RollupDef;
325
- /** Top 1000 keywords by clicks over the trailing 28-day window. */
326
- declare const topKeywords28dRollup: RollupDef;
327
310
  /**
328
311
  * Parquet-format companion to `topKeywords28dRollup`. Same shape, but persists
329
312
  * as a parquet object plus JSON sidecar pointer so widgets that need
@@ -485,4 +468,4 @@ declare const DEFAULT_ROLLUPS: readonly RollupDef[];
485
468
  * (CLI: `gscdump rollups --with-canonical`).
486
469
  */
487
470
  declare const CANONICAL_ROLLUPS: readonly RollupDef[];
488
- export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
471
+ export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup };