@gscdump/engine 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +53 -0
  3. package/dist/adapters/duckdb-node.d.mts +19 -0
  4. package/dist/adapters/duckdb-node.mjs +78 -0
  5. package/dist/adapters/filesystem.d.mts +206 -0
  6. package/dist/adapters/filesystem.mjs +320 -0
  7. package/dist/adapters/http.d.mts +227 -0
  8. package/dist/adapters/http.mjs +119 -0
  9. package/dist/adapters/hyparquet.d.mts +107 -0
  10. package/dist/adapters/hyparquet.mjs +250 -0
  11. package/dist/adapters/inspection-sqlite-browser.d.mts +9 -0
  12. package/dist/adapters/inspection-sqlite-browser.mjs +42 -0
  13. package/dist/adapters/inspection-sqlite-node.d.mts +9 -0
  14. package/dist/adapters/inspection-sqlite-node.mjs +32 -0
  15. package/dist/adapters/node-harness.d.mts +334 -0
  16. package/dist/adapters/node-harness.mjs +1907 -0
  17. package/dist/adapters/r2-manifest.d.mts +227 -0
  18. package/dist/adapters/r2-manifest.mjs +355 -0
  19. package/dist/adapters/r2.d.mts +93 -0
  20. package/dist/adapters/r2.mjs +65 -0
  21. package/dist/arrow-utils.d.mts +14 -0
  22. package/dist/arrow-utils.mjs +8 -0
  23. package/dist/contracts.d.mts +436 -0
  24. package/dist/contracts.mjs +1 -0
  25. package/dist/entities.d.mts +238 -0
  26. package/dist/entities.mjs +359 -0
  27. package/dist/index.d.mts +1849 -0
  28. package/dist/index.mjs +1976 -0
  29. package/dist/ingest.d.mts +96 -0
  30. package/dist/ingest.mjs +187 -0
  31. package/dist/planner.d.mts +16 -0
  32. package/dist/planner.mjs +321 -0
  33. package/dist/resolver/index.d.mts +207 -0
  34. package/dist/resolver/index.mjs +869 -0
  35. package/dist/rollups.d.mts +207 -0
  36. package/dist/rollups.mjs +553 -0
  37. package/dist/schema.d.mts +1258 -0
  38. package/dist/schema.mjs +139 -0
  39. package/dist/scope.d.mts +38 -0
  40. package/dist/scope.mjs +28 -0
  41. package/dist/snapshot.d.mts +14 -0
  42. package/dist/snapshot.mjs +1 -0
  43. package/dist/sql-bind.d.mts +19 -0
  44. package/dist/sql-bind.mjs +92 -0
  45. package/dist/sql-fragments.d.mts +21 -0
  46. package/dist/sql-fragments.mjs +13 -0
  47. package/package.json +168 -0
@@ -0,0 +1,207 @@
1
+ import { SQL } from "drizzle-orm";
2
+ import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities } from "gscdump/query/plan";
3
+ import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
4
+ import { TableName } from "gscdump/contracts";
5
+ type DimensionSurface = 'api' | 'stored' | 'derived';
6
+ interface DimensionBinding {
7
+ column: string;
8
+ surfaces: readonly DimensionSurface[];
9
+ }
10
+ interface LogicalDatasetDefinition {
11
+ dimensions: Partial<Record<Dimension, DimensionBinding>>;
12
+ }
13
+ declare const DIMENSION_SURFACES: Record<Dimension, readonly DimensionSurface[]>;
14
+ declare const LOGICAL_DATASETS: Record<LogicalDataset, LogicalDatasetDefinition>;
15
+ declare function inferLogicalDataset(dimensions: readonly Dimension[], filterDims?: readonly Dimension[]): LogicalDataset;
16
+ declare function dimensionColumn(dim: Dimension, dataset: LogicalDataset): string;
17
+ declare function supportsDimensionOnSurface(dim: Dimension, surface: DimensionSurface): boolean;
18
+ declare function assertDimensionsSupported(dimensions: readonly Dimension[], surface: DimensionSurface, context: string): void;
19
+ interface SqlFragmentsConfig<TableKey extends string> {
20
+ schema: Record<TableKey, unknown>;
21
+ datasetToTableKey: Record<keyof typeof LOGICAL_DATASETS, TableKey>;
22
+ metricCast: 'REAL' | 'DOUBLE';
23
+ regexPredicate: (expr: SQL, pattern: string, negate: boolean) => SQL;
24
+ tableLabel: string;
25
+ includeSiteId: boolean;
26
+ urlToPathExpr?: (col: string) => string;
27
+ /**
28
+ * Override the FROM-clause table reference. Default emits the bound drizzle
29
+ * table (e.g. `"pages"`). Parquet/R2 adapter overrides this to emit
30
+ * `read_parquet({{FILES}}, ...) AS "${tk}"` so the runSQL pipeline can swap
31
+ * in an object-key list while column refs (`"pages"."url"`) still resolve
32
+ * against the alias.
33
+ */
34
+ tableRef?: (tableKey: TableKey) => SQL;
35
+ }
36
+ interface SqlFragments<TableKey extends string> {
37
+ METRIC_NAMES: Metric[];
38
+ DIM_COLUMN_MAP: Record<TableKey, Record<string, string>>;
39
+ isMetricDimension: (dim: string) => dim is Metric;
40
+ tableKeyForDataset: (dataset: LogicalDataset$1) => TableKey;
41
+ dimColumn: (dim: Dimension, table: TableKey) => string;
42
+ inferTable: (dimensions: Dimension[], filterDims?: Dimension[]) => TableKey;
43
+ urlToPathExpr: (col: string) => string;
44
+ colRef: (tableKey: TableKey, colName: string) => SQL;
45
+ tableRef: (tableKey: TableKey) => SQL;
46
+ dateColRef: (tableKey: TableKey) => SQL;
47
+ siteIdColRef?: (tableKey: TableKey) => SQL;
48
+ dimExprSql: (dim: Dimension, tableKey: TableKey) => SQL;
49
+ metricSql: (metric: Metric, tableKey: TableKey) => SQL;
50
+ havingPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
51
+ dimensionPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
52
+ topLevelPredicate: (filters: InternalFilter[], tableKey: TableKey) => SQL | undefined;
53
+ }
54
+ declare function createSqlFragments<TableKey extends string>(config: SqlFragmentsConfig<TableKey>): SqlFragments<TableKey>;
55
+ interface ResolverAdapter<TableKey extends string = string> {
56
+ readonly METRIC_NAMES: readonly Metric[];
57
+ readonly capabilities: PlannerCapabilities;
58
+ readonly schema: Record<TableKey, unknown>;
59
+ tableKeyForDataset: (dataset: LogicalDataset$1) => TableKey;
60
+ inferTable: (dimensions: Dimension[], filterDims?: Dimension[]) => TableKey;
61
+ dimColumn: (dim: Dimension, tableKey: TableKey) => string;
62
+ isMetricDimension: (dim: string) => dim is Metric;
63
+ tableRef: (tableKey: TableKey) => SQL;
64
+ dateColRef: (tableKey: TableKey) => SQL;
65
+ urlToPathExpr: (col: string) => string;
66
+ siteIdColRef?: (tableKey: TableKey) => SQL;
67
+ dimExprSql: (dim: Dimension, tableKey: TableKey) => SQL;
68
+ metricSql: (metric: Metric, tableKey: TableKey) => SQL;
69
+ dimensionPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
70
+ havingPredicates: (filters: InternalFilter[], tableKey: TableKey) => SQL[];
71
+ topLevelPredicate: (filters: InternalFilter[], tableKey: TableKey) => SQL | undefined;
72
+ compile: (query: SQL) => {
73
+ sql: string;
74
+ params: unknown[];
75
+ };
76
+ }
77
+ type ComparisonFilter = 'new' | 'lost' | 'improving' | 'declining';
78
+ interface ResolverOptions<TableKey extends string = string> {
79
+ adapter: ResolverAdapter<TableKey>;
80
+ /** Optional site scope. Required for multi-tenant D1; omitted for parquet. */
81
+ siteId?: string | number;
82
+ }
83
+ interface ResolvedSQL {
84
+ sql: string;
85
+ params: unknown[];
86
+ countSql: string;
87
+ countParams: unknown[];
88
+ }
89
+ interface ResolvedSQLOptimized {
90
+ sql: string;
91
+ params: unknown[];
92
+ }
93
+ interface ResolvedComparisonSQL {
94
+ sql: string;
95
+ params: unknown[];
96
+ countSql: string;
97
+ countParams: unknown[];
98
+ }
99
+ interface ExtraQuery {
100
+ key: string;
101
+ sql: string;
102
+ params: unknown[];
103
+ }
104
+ interface CreateResolverAdapterConfig<TableKey extends string> extends SqlFragmentsConfig<TableKey> {
105
+ compile: (query: SQL) => {
106
+ sql: string;
107
+ params: unknown[];
108
+ };
109
+ capabilities: PlannerCapabilities;
110
+ }
111
+ declare function createResolverAdapter<TableKey extends string>(config: CreateResolverAdapterConfig<TableKey>): ResolverAdapter<TableKey>;
112
+ declare function resolveToSQLOptimized<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ResolvedSQLOptimized;
113
+ declare function resolveToSQL<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ResolvedSQL;
114
+ declare function buildTotalsSql<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): {
115
+ sql: string;
116
+ params: unknown[];
117
+ };
118
+ declare function resolveComparisonSQL<TK extends string>(current: BuilderState, previous: BuilderState, options: ResolverOptions<TK>, comparisonFilter?: ComparisonFilter): ResolvedComparisonSQL;
119
+ declare function buildExtrasQueries<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ExtraQuery[];
120
+ declare function mergeExtras(rows: Record<string, unknown>[], extrasResults: {
121
+ key: string;
122
+ results: Record<string, unknown>[];
123
+ }[]): Record<string, unknown>[];
124
+ type QueryRow = Record<string, unknown>;
125
+ interface FileSet {
126
+ table: TableName;
127
+ partitions: string[];
128
+ }
129
+ interface ExecuteSqlOptions {
130
+ fileSets?: Record<string, FileSet>;
131
+ }
132
+ interface SourceCapabilities extends PlannerCapabilities {
133
+ attachedTables?: boolean;
134
+ fileSets?: boolean;
135
+ localSource?: boolean;
136
+ }
137
+ interface RowQuerySource {
138
+ name?: string;
139
+ capabilities: SourceCapabilities;
140
+ queryRows: (state: BuilderState) => Promise<QueryRow[]>;
141
+ readonly executeSql?: undefined;
142
+ }
143
+ interface SqlQuerySource {
144
+ name?: string;
145
+ capabilities: SourceCapabilities;
146
+ queryRows: (state: BuilderState) => Promise<QueryRow[]>;
147
+ executeSql: (sql: string, params?: unknown[], opts?: ExecuteSqlOptions) => Promise<QueryRow[]>;
148
+ }
149
+ type AnalysisQuerySource = RowQuerySource | SqlQuerySource;
150
+ declare function isSqlQuerySource(s: AnalysisQuerySource): s is SqlQuerySource;
151
+ interface CreateSqlQuerySourceOptions<TKey extends string> {
152
+ /** Debug-only identifier surfaced on the source for error messages. */
153
+ name: string;
154
+ /** Dialect-specific adapter; compiles `BuilderState` → `{ sql, params }`. */
155
+ adapter: ResolverAdapter<TKey>;
156
+ /** Drives the underlying DB. Called for both typed queries and raw SQL. */
157
+ execute: (sql: string, params: unknown[]) => Promise<QueryRow[]>;
158
+ /** Tenant id for multi-tenant dialects; forwarded to `resolveToSQL`. */
159
+ siteId?: string | number;
160
+ /** Additional capability flags merged on top of `adapter.capabilities`. */
161
+ extraCapabilities?: Partial<SourceCapabilities>;
162
+ }
163
+ declare function createSqlQuerySource<TKey extends string>(options: CreateSqlQuerySourceOptions<TKey>): SqlQuerySource;
164
+ declare function compilePg(query: SQL): {
165
+ sql: string;
166
+ params: unknown[];
167
+ };
168
+ declare function compileSqlite(query: SQL): {
169
+ sql: string;
170
+ params: unknown[];
171
+ };
172
+ declare function getInternalFilters(filter: FilterInput | undefined): InternalFilter[];
173
+ declare function getDimensionFilters(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): InternalFilter[];
174
+ declare function getFilterDimensions(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): Dimension[];
175
+ declare function metricValue(row: Record<string, unknown>, metric: string): number;
176
+ declare function dimensionValue(row: Record<string, unknown>, dimension: string): string;
177
+ declare function matchesDimensionFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
178
+ declare function matchesMetricFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
179
+ declare function matchesTopLevelPage(row: Record<string, unknown>): boolean;
180
+ type PgTableKey = TableName;
181
+ declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
182
+ /**
183
+ * Parquet-aware variant of {@link pgResolverAdapter}. Identical SQL output
184
+ * except FROM clauses emit `read_parquet({{FILES}}, union_by_name = true) AS
185
+ * "${tk}"`. The runSQL pipeline substitutes `{{FILES}}` with R2 object keys
186
+ * resolved from the manifest. The `AS "${tk}"` alias is mandatory — drizzle
187
+ * compiles `colRef` to table-qualified `"pages"."url"`, which would not
188
+ * resolve against an unaliased FROM.
189
+ *
190
+ * Single-use: build a fresh adapter per query. Cheap (no I/O) and avoids
191
+ * accidental adapter caching that would lock in a stale `{{FILES}}` set.
192
+ */
193
+ declare function createParquetResolverAdapter(): ResolverAdapter<PgTableKey>;
194
+ interface AssertSchemaInSyncOptions {
195
+ /** Label used in the thrown error (e.g. 'browser', 'sqlite'). */
196
+ label: string;
197
+ /** Drizzle schema keyed by table name. For sqlite, strip the `gsc_` prefix before passing. */
198
+ schema: Record<string, unknown>;
199
+ /**
200
+ * Map drizzle table key → canonical `TableName`. For browser this is identity;
201
+ * for sqlite this strips the `gsc_` prefix.
202
+ */
203
+ tableKeyToName: (key: string) => TableName;
204
+ mode: 'exact' | 'superset';
205
+ }
206
+ declare function assertSchemaInSync(options: AssertSchemaInSyncOptions): void;
207
+ export { type AnalysisQuerySource, type AssertSchemaInSyncOptions, type ComparisonFilter, type CreateResolverAdapterConfig, type CreateSqlQuerySourceOptions, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExecuteSqlOptions, type ExtraQuery, type FileSet, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type PgTableKey, type QueryRow, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverOptions, type RowQuerySource, type SourceCapabilities, type SqlFragments, type SqlFragmentsConfig, type SqlQuerySource, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, compilePg, compileSqlite, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, createSqlQuerySource, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isSqlQuerySource, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, supportsDimensionOnSurface };