@gscdump/engine 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,104 @@
1
- import { AnalysisQuerySource, AnalysisSourceKind, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
2
- import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/index.mjs";
1
+ import { Row, SearchType as SearchType$1, StorageEngine, TenantCtx } from "../_chunks/storage.mjs";
2
+ import { AnalysisParams, AnalysisResult } from "../_chunks/analysis-types.mjs";
3
+ import { ResolverAdapter } from "../_chunks/types.mjs";
4
+ import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
5
+ import { EngineError } from "../_chunks/errors.mjs";
6
+ import "../_chunks/contracts.mjs";
7
+ import { PlannerCapabilities } from "gscdump/query/plan";
8
+ import { BuilderState } from "gscdump/query";
9
+ interface AttachedTableRunner {
10
+ /**
11
+ * Run a query with positional (`?`) bound parameters. Return objects keyed
12
+ * by column name. BIGINT → number coercion is applied by the source factory
13
+ * (see `coerceRows`); runners only need to handle DATE → ISO string (or
14
+ * let the analyzer reducer normalize via `num(v)`/`str(v)`).
15
+ */
16
+ query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<Row[]>;
17
+ }
18
+ interface AttachedTableSourceOptions {
19
+ /** Schema name the exported DuckDB file was attached under — e.g. `gsc`. */
20
+ schema: string;
21
+ /**
22
+ * Abort in-flight queries when the caller no longer cares about the
23
+ * result. Every `runner.query` call receives the same signal.
24
+ */
25
+ signal?: AbortSignal;
26
+ /**
27
+ * List of table names actually attached to this connection. When provided,
28
+ * `executeSql` short-circuits with a specific "table not attached" error
29
+ * if the SQL plan references a table that isn't in this list — letting
30
+ * callers (e.g. the analytics layer) route to cloud fallback without
31
+ * paying the SQL execution cost. Omit to disable the check.
32
+ */
33
+ attachedTables?: readonly string[];
34
+ /**
35
+ * Dialect adapter surfaced on the source for analyzers that compose SQL
36
+ * from a `BuilderState` at plan-build time (e.g. `data-query`,
37
+ * `data-detail`). Attached-table sources execute pg-flavored DuckDB SQL,
38
+ * so callers should pass `pgResolverAdapter` here.
39
+ */
40
+ adapter?: ResolverAdapter<any>;
41
+ }
42
+ declare class AttachedTableMissingError extends Error {
43
+ readonly missing: readonly string[];
44
+ readonly engineError: EngineError;
45
+ constructor(missing: readonly string[]);
46
+ }
47
+ declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
48
+ interface CreateSqlQuerySourceOptions<TKey extends string> {
49
+ /** Debug-only identifier surfaced on the source for error messages. */
50
+ name: string;
51
+ /** Telemetry tag stamped onto analyzer result meta. */
52
+ kind?: AnalysisSourceKind;
53
+ /** Dialect-specific adapter; compiles `BuilderState` → `{ sql, params }`. */
54
+ adapter: ResolverAdapter<TKey>;
55
+ /** Drives the underlying DB. Called for both typed queries and raw SQL. */
56
+ execute: (sql: string, params: unknown[]) => Promise<QueryRow[]>;
57
+ /** Tenant id for multi-tenant dialects; forwarded to `resolveToSQL`. */
58
+ siteId?: string | number;
59
+ /**
60
+ * Search-type scope for multi-tenant dialects; forwarded to `resolveToSQL`.
61
+ * `number` = int-encoded code (`SEARCH_TYPE_INT`) for INT `search_type`
62
+ * catalogs (bound bare so the int partition prunes); `string` otherwise.
63
+ */
64
+ searchType?: string | number;
65
+ /** Additional capability flags merged on top of `adapter.capabilities`. */
66
+ extraCapabilities?: Partial<SourceCapabilities>;
67
+ }
68
+ declare function createSqlQuerySource<TKey extends string>(options: CreateSqlQuerySourceOptions<TKey>): AnalysisQuerySource;
69
+ /**
70
+ * Capabilities the engine query path honors. Matches what the DuckDB compiler
71
+ * passes to `buildLogicalPlan`: regex pushes down; comparison joins and
72
+ * multi-dataset queries belong to the analyzer dispatcher, not the engine's
73
+ * builder-state query path.
74
+ */
75
+ declare const ENGINE_QUERY_CAPABILITIES: PlannerCapabilities;
76
+ interface EngineQuerySourceOptions {
77
+ engine: StorageEngine;
78
+ ctx: TenantCtx;
79
+ /**
80
+ * Restrict every manifest lookup the source performs to a single search-type
81
+ * slice. Threads into `engine.query` and `engine.runSQL` so the wrapped
82
+ * source returns rows from one cohort instead of unioning web + non-web
83
+ * parquet. Undefined preserves legacy cross-type behaviour for web-only
84
+ * tenants and admin paths.
85
+ */
86
+ searchType?: SearchType$1;
87
+ }
88
+ /**
89
+ * Wraps a storage engine as an `AnalysisQuerySource` with SQL execution.
90
+ * `queryRows` runs typed builder-state queries; `executeSql` delegates to
91
+ * `engine.runSQL` and requires `opts.fileSets` (with a `FILES` entry so the
92
+ * target table can be resolved for partition lookup).
93
+ */
94
+ declare function createEngineQuerySource(options: EngineQuerySourceOptions): AnalysisQuerySource;
95
+ /**
96
+ * Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
97
+ * Equivalent to wrapping `createEngineQuerySource`, with omitted searchType
98
+ * defaulted to web at this public helper boundary.
99
+ */
100
+ declare function runAnalyzerWithEngine(deps: {
101
+ engine: StorageEngine;
102
+ }, ctx: TenantCtx, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
103
+ declare function queryRows<TRow = QueryRow>(source: AnalysisQuerySource, state: BuilderState): Promise<TRow[]>;
3
104
  export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
@@ -1,2 +1,143 @@
1
- import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/source.mjs";
1
+ import { engineErrors } from "../errors.mjs";
2
+ import "../_chunks/layout.mjs";
3
+ import { coerceRows } from "../_chunks/coerce.mjs";
4
+ import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "../_chunks/resolver.mjs";
5
+ import { runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
6
+ var AttachedTableMissingError = class extends Error {
7
+ missing;
8
+ engineError;
9
+ constructor(missing) {
10
+ const engineError = engineErrors.attachedTableMissing(missing);
11
+ super(engineError.message);
12
+ this.missing = missing;
13
+ this.name = "AttachedTableMissingError";
14
+ this.engineError = engineError;
15
+ }
16
+ };
17
+ const ATTACHED_TABLE_CAPABILITIES = {
18
+ fileSets: true,
19
+ attachedTables: true,
20
+ regex: true
21
+ };
22
+ const ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER = {
23
+ ...ATTACHED_TABLE_CAPABILITIES,
24
+ adapter: true
25
+ };
26
+ function rewriteForTableSource(sql, schema, fileSets) {
27
+ let out = sql;
28
+ for (const [key, fs] of Object.entries(fileSets)) {
29
+ const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${key}\\}\\}\\s*,\\s*union_by_name\\s*=\\s*true\\s*\\)`, "g");
30
+ out = out.replace(pattern, `${schema}.${fs.table}`);
31
+ }
32
+ return out;
33
+ }
34
+ function createAttachedTableSource(runner, options) {
35
+ const { schema, signal, attachedTables, adapter } = options;
36
+ const attachedSet = attachedTables ? new Set(attachedTables) : null;
37
+ return {
38
+ name: "attached-table",
39
+ kind: "browser",
40
+ capabilities: adapter ? ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER : ATTACHED_TABLE_CAPABILITIES,
41
+ adapter,
42
+ async queryRows() {
43
+ throw new Error("attached-table source: queryRows is not supported; use SQL analyzers");
44
+ },
45
+ async executeSql(sql, params, opts) {
46
+ signal?.throwIfAborted();
47
+ const fileSets = opts?.fileSets ?? {};
48
+ if (attachedSet) {
49
+ const missing = [];
50
+ for (const fs of Object.values(fileSets)) if (!attachedSet.has(fs.table)) missing.push(fs.table);
51
+ if (missing.length > 0) throw new AttachedTableMissingError(missing);
52
+ }
53
+ const rewritten = rewriteForTableSource(sql, schema, fileSets);
54
+ return coerceRows(await runner.query(rewritten, params ?? [], signal));
55
+ }
56
+ };
57
+ }
58
+ function createSqlQuerySource(options) {
59
+ const { name, kind, adapter, execute, siteId, searchType, extraCapabilities } = options;
60
+ return {
61
+ name,
62
+ kind,
63
+ capabilities: {
64
+ ...adapter.capabilities,
65
+ ...extraCapabilities,
66
+ adapter: true
67
+ },
68
+ adapter,
69
+ siteId,
70
+ async queryRows(state) {
71
+ const resolved = resolveToSQL(state, {
72
+ adapter,
73
+ siteId,
74
+ searchType
75
+ });
76
+ return coerceRows(await execute(resolved.sql, resolved.params));
77
+ },
78
+ async executeSql(sql, params) {
79
+ return coerceRows(await execute(sql, params ?? []));
80
+ }
81
+ };
82
+ }
83
+ function isMetricDimension(dim) {
84
+ return [
85
+ "clicks",
86
+ "impressions",
87
+ "ctr",
88
+ "position"
89
+ ].includes(dim);
90
+ }
91
+ const ENGINE_QUERY_CAPABILITIES = {
92
+ regex: true,
93
+ multiDataset: false,
94
+ comparisonJoin: false,
95
+ windowTotals: false
96
+ };
97
+ const ENGINE_SOURCE_CAPABILITIES = {
98
+ ...ENGINE_QUERY_CAPABILITIES,
99
+ fileSets: true,
100
+ adapter: true
101
+ };
102
+ function createEngineQuerySource(options) {
103
+ const { engine, ctx, searchType } = options;
104
+ return {
105
+ name: "engine",
106
+ kind: "local",
107
+ capabilities: ENGINE_SOURCE_CAPABILITIES,
108
+ adapter: pgResolverAdapter,
109
+ async queryRows(state) {
110
+ const filterDims = getFilterDimensions(state.filter, isMetricDimension);
111
+ assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
112
+ if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
113
+ return coerceRows((await engine.query({
114
+ ...ctx,
115
+ ...searchType !== void 0 ? { searchType } : {}
116
+ }, state)).rows);
117
+ },
118
+ async executeSql(sql, params, opts) {
119
+ const fileSets = opts?.fileSets;
120
+ if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
121
+ const { rows } = await engine.runSQL({
122
+ ctx,
123
+ table: fileSets.FILES.table,
124
+ fileSets,
125
+ sql,
126
+ params: params ?? [],
127
+ ...searchType !== void 0 ? { searchType } : {}
128
+ });
129
+ return coerceRows(rows);
130
+ }
131
+ };
132
+ }
133
+ async function runAnalyzerWithEngine(deps, ctx, params, registry) {
134
+ return runAnalyzerFromSource(createEngineQuerySource({
135
+ engine: deps.engine,
136
+ ctx,
137
+ searchType: params.searchType ?? "web"
138
+ }), params, registry);
139
+ }
140
+ async function queryRows(source, state) {
141
+ return await source.queryRows(state);
142
+ }
2
143
  export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/engine",
3
3
  "type": "module",
4
- "version": "1.0.2",
4
+ "version": "1.0.4",
5
5
  "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -51,6 +51,11 @@
51
51
  "import": "./dist/ingest.mjs",
52
52
  "default": "./dist/ingest.mjs"
53
53
  },
54
+ "./ingest-accumulator": {
55
+ "types": "./dist/ingest-accumulator.d.mts",
56
+ "import": "./dist/ingest-accumulator.mjs",
57
+ "default": "./dist/ingest-accumulator.mjs"
58
+ },
54
59
  "./sink-node": {
55
60
  "types": "./dist/sink-node.d.mts",
56
61
  "import": "./dist/sink-node.mjs",
@@ -71,6 +76,11 @@
71
76
  "import": "./dist/entities.mjs",
72
77
  "default": "./dist/entities.mjs"
73
78
  },
79
+ "./entity-keys": {
80
+ "types": "./dist/entity-keys.d.mts",
81
+ "import": "./dist/entity-keys.mjs",
82
+ "default": "./dist/entity-keys.mjs"
83
+ },
74
84
  "./rollups": {
75
85
  "types": "./dist/rollups.d.mts",
76
86
  "import": "./dist/rollups.mjs",
@@ -166,9 +176,9 @@
166
176
  "dependencies": {
167
177
  "drizzle-orm": "1.0.0-rc.3",
168
178
  "proper-lockfile": "^4.1.2",
169
- "@gscdump/contracts": "^1.0.2",
170
- "@gscdump/lakehouse": "^1.0.2",
171
- "gscdump": "^1.0.2"
179
+ "@gscdump/lakehouse": "^1.0.4",
180
+ "gscdump": "^1.0.4",
181
+ "@gscdump/contracts": "^1.0.4"
172
182
  },
173
183
  "devDependencies": {
174
184
  "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
@@ -1,27 +0,0 @@
1
- import { ParquetCodec, QueryExecutor, Row } from "./storage.mjs";
2
- interface DuckDBHandle {
3
- query: (sql: string, params?: unknown[]) => Promise<Row[]>;
4
- registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
5
- copyFileToBuffer: (name: string) => Promise<Uint8Array>;
6
- dropFiles: (names: string[]) => Promise<void>;
7
- /**
8
- * Returns a unique path suitable for `COPY TO '…'` + `copyFileToBuffer`.
9
- * In Node this is an absolute path under `os.tmpdir()` so DuckDB doesn't
10
- * litter the CWD; in browsers/Workers it's a plain virtual-FS name.
11
- */
12
- makeTempPath: (ext: string) => string;
13
- }
14
- interface DuckDBFactory {
15
- getDuckDB: () => Promise<DuckDBHandle>;
16
- }
17
- interface DuckDBReadOptions {
18
- /**
19
- * Maximum non-URI parquet reads held in flight while registering DuckDB
20
- * buffers. Defaults to 16; callers handling unusually large files can lower
21
- * it to trade latency for peak JS memory.
22
- */
23
- bufferReadConcurrency?: number;
24
- }
25
- declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
26
- declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
27
- export { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor };
@@ -1,53 +0,0 @@
1
- import { AnalysisParams } from "./analysis-types.mjs";
2
- type WindowPreset = 'last-7d' | 'last-28d' | 'last-30d' | 'last-90d' | 'last-180d' | 'last-365d' | 'mtd' | 'ytd' | 'custom';
3
- type ComparisonMode = 'none' | 'prev-period' | 'yoy';
4
- interface ResolveWindowOptions {
5
- preset: WindowPreset;
6
- comparison?: ComparisonMode;
7
- anchor?: string;
8
- start?: string;
9
- end?: string;
10
- }
11
- interface ResolvedWindow {
12
- start: string;
13
- end: string;
14
- days: number;
15
- comparison?: {
16
- start: string;
17
- end: string;
18
- };
19
- }
20
- interface AnalysisPeriod {
21
- startDate: string;
22
- endDate: string;
23
- }
24
- interface ComparisonPeriod {
25
- current: AnalysisPeriod;
26
- previous: AnalysisPeriod;
27
- }
28
- declare function defaultEndDate(): string;
29
- declare function periodOf(params: AnalysisParams): AnalysisPeriod;
30
- declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
31
- declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
32
- interface PadTimeseriesOptions<T> {
33
- /** ISO date (YYYY-MM-DD), inclusive lower bound. */
34
- startDate: string;
35
- /** ISO date (YYYY-MM-DD), inclusive upper bound. */
36
- endDate: string;
37
- /**
38
- * Row to insert for missing dates. Defaults to `{ clicks: 0, impressions: 0, ctr: 0, position: 0 }`.
39
- * The `date` field is set automatically.
40
- */
41
- fill?: Omit<T, 'date'>;
42
- /** Row-field that carries the ISO date. Defaults to `date`. */
43
- dateKey?: string;
44
- }
45
- type DateRowShape = Record<string, unknown> & {
46
- date?: unknown;
47
- };
48
- /**
49
- * Pad rows so every calendar day in `[startDate, endDate]` appears at least
50
- * once. Existing dates keep all their rows (grouped timeseries safe).
51
- */
52
- declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
53
- export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
@@ -1,56 +0,0 @@
1
- import { ResolverAdapter } from "./types.mjs";
2
- import { TableName } from "@gscdump/contracts";
3
- type PgTableKey = TableName;
4
- declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
5
- /**
6
- * Parquet-aware variant of {@link pgResolverAdapter}. Identical SQL output
7
- * except FROM clauses emit `read_parquet({{FILES}}, union_by_name = true) AS
8
- * "${tk}"`. The runSQL pipeline substitutes `{{FILES}}` with R2 object keys
9
- * resolved from the manifest. The `AS "${tk}"` alias is mandatory — drizzle
10
- * compiles `colRef` to table-qualified `"pages"."url"`, which would not
11
- * resolve against an unaliased FROM.
12
- *
13
- * Single-use: build a fresh adapter per query. Cheap (no I/O) and avoids
14
- * accidental adapter caching that would lock in a stale `{{FILES}}` set.
15
- */
16
- interface ResolverAdapterOptions {
17
- /**
18
- * `queryDim` reads canonical from a joined query dimension. `column` is only
19
- * for derived canonical rollup relations whose primary relation already
20
- * carries a null-free `query_canonical` output column.
21
- */
22
- queryCanonicalSource?: 'queryDim' | 'column';
23
- }
24
- interface R2SqlResolverAdapterOptions extends ResolverAdapterOptions {
25
- /**
26
- * R2 SQL string partition equality can undercount on identity partitions;
27
- * string-encoded catalogs use CONCAT(col, '') in partition predicates. Int
28
- * catalogs do not need the workaround and keep bare equality for pruning.
29
- */
30
- partitionKeyEncoding?: 'string' | 'int';
31
- }
32
- declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
33
- /**
34
- * Multi-tenant pg-flavored adapter for the Iceberg / R2 SQL read path.
35
- * Identical SQL output to `pgResolverAdapter` except WHERE clauses inject
36
- * `site_id = ?` AND `search_type = ?` automatically when those scopes are
37
- * passed to `resolveToSQL`. Required for the Iceberg fact tables which are
38
- * shared across tenants — querying without these predicates would leak
39
- * cross-tenant data. Single-use: the adapter has no `tableRef` override,
40
- * so callers must rewrite bare table names to their qualified form (e.g.
41
- * `${namespace}.pages`) before sending to R2 SQL.
42
- */
43
- declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
44
- /**
45
- * R2 SQL adapter for the Iceberg fact tables.
46
- *
47
- * It shares the multi-tenant Iceberg schema with `createIcebergResolverAdapter`
48
- * but models R2 SQL's narrower execution surface: no window-total plans and no
49
- * comparison joins. Int-partition catalogs are the default and emit bare
50
- * equality predicates for pruning. Legacy string-partition catalogs must pass
51
- * `partitionKeyEncoding: 'string'` to emit `CONCAT(partition_col, '') = ?`,
52
- * working around R2 SQL's partition-string equality undercount while preserving
53
- * bound params.
54
- */
55
- declare function createR2SqlResolverAdapter(options?: R2SqlResolverAdapterOptions): ResolverAdapter<PgTableKey>;
56
- export { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter };
@@ -1,15 +0,0 @@
1
- import { TableName } from "./storage.mjs";
2
- import { LogicalQueryPlan } from "gscdump/query/plan";
3
- import { BuilderState } from "gscdump/query";
4
- interface ResolvedQuery {
5
- sql: string;
6
- params: unknown[];
7
- partitions: string[];
8
- table: TableName;
9
- filesPlaceholder: string;
10
- }
11
- declare const FILES_PLACEHOLDER = "{{FILES}}";
12
- declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
13
- declare function resolveParquetSQL(state: BuilderState, table?: TableName): ResolvedQuery;
14
- declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
15
- export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles };
@@ -1,143 +0,0 @@
1
- import { engineErrors } from "../errors.mjs";
2
- import "./layout.mjs";
3
- import { coerceRows } from "./coerce.mjs";
4
- import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "./resolver.mjs";
5
- import { runAnalyzerFromSource } from "./dispatch.mjs";
6
- var AttachedTableMissingError = class extends Error {
7
- missing;
8
- engineError;
9
- constructor(missing) {
10
- const engineError = engineErrors.attachedTableMissing(missing);
11
- super(engineError.message);
12
- this.missing = missing;
13
- this.name = "AttachedTableMissingError";
14
- this.engineError = engineError;
15
- }
16
- };
17
- const ATTACHED_TABLE_CAPABILITIES = {
18
- fileSets: true,
19
- attachedTables: true,
20
- regex: true
21
- };
22
- const ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER = {
23
- ...ATTACHED_TABLE_CAPABILITIES,
24
- adapter: true
25
- };
26
- function rewriteForTableSource(sql, schema, fileSets) {
27
- let out = sql;
28
- for (const [key, fs] of Object.entries(fileSets)) {
29
- const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${key}\\}\\}\\s*,\\s*union_by_name\\s*=\\s*true\\s*\\)`, "g");
30
- out = out.replace(pattern, `${schema}.${fs.table}`);
31
- }
32
- return out;
33
- }
34
- function createAttachedTableSource(runner, options) {
35
- const { schema, signal, attachedTables, adapter } = options;
36
- const attachedSet = attachedTables ? new Set(attachedTables) : null;
37
- return {
38
- name: "attached-table",
39
- kind: "browser",
40
- capabilities: adapter ? ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER : ATTACHED_TABLE_CAPABILITIES,
41
- adapter,
42
- async queryRows() {
43
- throw new Error("attached-table source: queryRows is not supported; use SQL analyzers");
44
- },
45
- async executeSql(sql, params, opts) {
46
- signal?.throwIfAborted();
47
- const fileSets = opts?.fileSets ?? {};
48
- if (attachedSet) {
49
- const missing = [];
50
- for (const fs of Object.values(fileSets)) if (!attachedSet.has(fs.table)) missing.push(fs.table);
51
- if (missing.length > 0) throw new AttachedTableMissingError(missing);
52
- }
53
- const rewritten = rewriteForTableSource(sql, schema, fileSets);
54
- return coerceRows(await runner.query(rewritten, params ?? [], signal));
55
- }
56
- };
57
- }
58
- function createSqlQuerySource(options) {
59
- const { name, kind, adapter, execute, siteId, searchType, extraCapabilities } = options;
60
- return {
61
- name,
62
- kind,
63
- capabilities: {
64
- ...adapter.capabilities,
65
- ...extraCapabilities,
66
- adapter: true
67
- },
68
- adapter,
69
- siteId,
70
- async queryRows(state) {
71
- const resolved = resolveToSQL(state, {
72
- adapter,
73
- siteId,
74
- searchType
75
- });
76
- return coerceRows(await execute(resolved.sql, resolved.params));
77
- },
78
- async executeSql(sql, params) {
79
- return coerceRows(await execute(sql, params ?? []));
80
- }
81
- };
82
- }
83
- function isMetricDimension(dim) {
84
- return [
85
- "clicks",
86
- "impressions",
87
- "ctr",
88
- "position"
89
- ].includes(dim);
90
- }
91
- const ENGINE_QUERY_CAPABILITIES = {
92
- regex: true,
93
- multiDataset: false,
94
- comparisonJoin: false,
95
- windowTotals: false
96
- };
97
- const ENGINE_SOURCE_CAPABILITIES = {
98
- ...ENGINE_QUERY_CAPABILITIES,
99
- fileSets: true,
100
- adapter: true
101
- };
102
- function createEngineQuerySource(options) {
103
- const { engine, ctx, searchType } = options;
104
- return {
105
- name: "engine",
106
- kind: "local",
107
- capabilities: ENGINE_SOURCE_CAPABILITIES,
108
- adapter: pgResolverAdapter,
109
- async queryRows(state) {
110
- const filterDims = getFilterDimensions(state.filter, isMetricDimension);
111
- assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
112
- if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
113
- return coerceRows((await engine.query({
114
- ...ctx,
115
- ...searchType !== void 0 ? { searchType } : {}
116
- }, state)).rows);
117
- },
118
- async executeSql(sql, params, opts) {
119
- const fileSets = opts?.fileSets;
120
- if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
121
- const { rows } = await engine.runSQL({
122
- ctx,
123
- table: fileSets.FILES.table,
124
- fileSets,
125
- sql,
126
- params: params ?? [],
127
- ...searchType !== void 0 ? { searchType } : {}
128
- });
129
- return coerceRows(rows);
130
- }
131
- };
132
- }
133
- async function runAnalyzerWithEngine(deps, ctx, params, registry) {
134
- return runAnalyzerFromSource(createEngineQuerySource({
135
- engine: deps.engine,
136
- ctx,
137
- searchType: params.searchType ?? "web"
138
- }), params, registry);
139
- }
140
- async function queryRows(source, state) {
141
- return await source.queryRows(state);
142
- }
143
- export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
@@ -1,7 +0,0 @@
1
- import { DuckDBHandle } from "../_chunks/duckdb.mjs";
2
- interface NodeDuckDBOptions {
3
- verbose?: boolean;
4
- }
5
- declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
6
- declare function resetNodeDuckDB(): void;
7
- export { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB };