@gscdump/engine 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/_chunks/coerce.mjs +1 -1
- package/dist/_chunks/engine.mjs +1 -1
- package/dist/_chunks/entities.mjs +3 -91
- package/dist/_chunks/index.d.mts +44 -95
- package/dist/_chunks/resolver.mjs +105 -105
- package/dist/_chunks/schema2.mjs +2 -1
- package/dist/_chunks/sink.d.mts +2 -1
- package/dist/_chunks/snapshot.d.mts +27 -1
- package/dist/adapters/node.d.mts +6 -2
- package/dist/adapters/node.mjs +93 -2
- package/dist/entities.d.mts +3 -36
- package/dist/entities.mjs +4 -2
- package/dist/entity-keys.d.mts +27 -0
- package/dist/entity-keys.mjs +93 -0
- package/dist/iceberg/index.mjs +7 -7
- package/dist/index.d.mts +6 -107
- package/dist/index.mjs +101 -132
- package/dist/ingest-accumulator.d.mts +97 -0
- package/dist/ingest-accumulator.mjs +123 -0
- package/dist/period/index.d.mts +1 -1
- package/dist/planner.d.mts +14 -2
- package/dist/report/index.d.mts +1 -1
- package/dist/resolver/index.d.mts +53 -1
- package/dist/rollups.mjs +3 -2
- package/dist/source/index.d.mts +103 -2
- package/dist/source/index.mjs +142 -1
- package/package.json +14 -4
- package/dist/_chunks/duckdb.d.mts +0 -27
- package/dist/_chunks/index2.d.mts +0 -53
- package/dist/_chunks/pg-adapter.d.mts +0 -56
- package/dist/_chunks/planner.d.mts +0 -15
- package/dist/_chunks/source.mjs +0 -143
- package/dist/adapters/duckdb-node.d.mts +0 -7
- package/dist/adapters/duckdb-node.mjs +0 -94
- package/dist/schedule.mjs +0 -100
- /package/dist/{schedule.d.mts → _chunks/schedule.d.mts} +0 -0
package/dist/source/index.d.mts
CHANGED
|
@@ -1,3 +1,104 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { Row, SearchType as SearchType$1, StorageEngine, TenantCtx } from "../_chunks/storage.mjs";
|
|
2
|
+
import { AnalysisParams, AnalysisResult } from "../_chunks/analysis-types.mjs";
|
|
3
|
+
import { ResolverAdapter } from "../_chunks/types.mjs";
|
|
4
|
+
import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
|
|
5
|
+
import { EngineError } from "../_chunks/errors.mjs";
|
|
6
|
+
import "../_chunks/contracts.mjs";
|
|
7
|
+
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
8
|
+
import { BuilderState } from "gscdump/query";
|
|
9
|
+
interface AttachedTableRunner {
|
|
10
|
+
/**
|
|
11
|
+
* Run a query with positional (`?`) bound parameters. Return objects keyed
|
|
12
|
+
* by column name. BIGINT → number coercion is applied by the source factory
|
|
13
|
+
* (see `coerceRows`); runners only need to handle DATE → ISO string (or
|
|
14
|
+
* let the analyzer reducer normalize via `num(v)`/`str(v)`).
|
|
15
|
+
*/
|
|
16
|
+
query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<Row[]>;
|
|
17
|
+
}
|
|
18
|
+
interface AttachedTableSourceOptions {
|
|
19
|
+
/** Schema name the exported DuckDB file was attached under — e.g. `gsc`. */
|
|
20
|
+
schema: string;
|
|
21
|
+
/**
|
|
22
|
+
* Abort in-flight queries when the caller no longer cares about the
|
|
23
|
+
* result. Every `runner.query` call receives the same signal.
|
|
24
|
+
*/
|
|
25
|
+
signal?: AbortSignal;
|
|
26
|
+
/**
|
|
27
|
+
* List of table names actually attached to this connection. When provided,
|
|
28
|
+
* `executeSql` short-circuits with a specific "table not attached" error
|
|
29
|
+
* if the SQL plan references a table that isn't in this list — letting
|
|
30
|
+
* callers (e.g. the analytics layer) route to cloud fallback without
|
|
31
|
+
* paying the SQL execution cost. Omit to disable the check.
|
|
32
|
+
*/
|
|
33
|
+
attachedTables?: readonly string[];
|
|
34
|
+
/**
|
|
35
|
+
* Dialect adapter surfaced on the source for analyzers that compose SQL
|
|
36
|
+
* from a `BuilderState` at plan-build time (e.g. `data-query`,
|
|
37
|
+
* `data-detail`). Attached-table sources execute pg-flavored DuckDB SQL,
|
|
38
|
+
* so callers should pass `pgResolverAdapter` here.
|
|
39
|
+
*/
|
|
40
|
+
adapter?: ResolverAdapter<any>;
|
|
41
|
+
}
|
|
42
|
+
declare class AttachedTableMissingError extends Error {
|
|
43
|
+
readonly missing: readonly string[];
|
|
44
|
+
readonly engineError: EngineError;
|
|
45
|
+
constructor(missing: readonly string[]);
|
|
46
|
+
}
|
|
47
|
+
declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
|
|
48
|
+
interface CreateSqlQuerySourceOptions<TKey extends string> {
|
|
49
|
+
/** Debug-only identifier surfaced on the source for error messages. */
|
|
50
|
+
name: string;
|
|
51
|
+
/** Telemetry tag stamped onto analyzer result meta. */
|
|
52
|
+
kind?: AnalysisSourceKind;
|
|
53
|
+
/** Dialect-specific adapter; compiles `BuilderState` → `{ sql, params }`. */
|
|
54
|
+
adapter: ResolverAdapter<TKey>;
|
|
55
|
+
/** Drives the underlying DB. Called for both typed queries and raw SQL. */
|
|
56
|
+
execute: (sql: string, params: unknown[]) => Promise<QueryRow[]>;
|
|
57
|
+
/** Tenant id for multi-tenant dialects; forwarded to `resolveToSQL`. */
|
|
58
|
+
siteId?: string | number;
|
|
59
|
+
/**
|
|
60
|
+
* Search-type scope for multi-tenant dialects; forwarded to `resolveToSQL`.
|
|
61
|
+
* `number` = int-encoded code (`SEARCH_TYPE_INT`) for INT `search_type`
|
|
62
|
+
* catalogs (bound bare so the int partition prunes); `string` otherwise.
|
|
63
|
+
*/
|
|
64
|
+
searchType?: string | number;
|
|
65
|
+
/** Additional capability flags merged on top of `adapter.capabilities`. */
|
|
66
|
+
extraCapabilities?: Partial<SourceCapabilities>;
|
|
67
|
+
}
|
|
68
|
+
declare function createSqlQuerySource<TKey extends string>(options: CreateSqlQuerySourceOptions<TKey>): AnalysisQuerySource;
|
|
69
|
+
/**
|
|
70
|
+
* Capabilities the engine query path honors. Matches what the DuckDB compiler
|
|
71
|
+
* passes to `buildLogicalPlan`: regex pushes down; comparison joins and
|
|
72
|
+
* multi-dataset queries belong to the analyzer dispatcher, not the engine's
|
|
73
|
+
* builder-state query path.
|
|
74
|
+
*/
|
|
75
|
+
declare const ENGINE_QUERY_CAPABILITIES: PlannerCapabilities;
|
|
76
|
+
interface EngineQuerySourceOptions {
|
|
77
|
+
engine: StorageEngine;
|
|
78
|
+
ctx: TenantCtx;
|
|
79
|
+
/**
|
|
80
|
+
* Restrict every manifest lookup the source performs to a single search-type
|
|
81
|
+
* slice. Threads into `engine.query` and `engine.runSQL` so the wrapped
|
|
82
|
+
* source returns rows from one cohort instead of unioning web + non-web
|
|
83
|
+
* parquet. Undefined preserves legacy cross-type behaviour for web-only
|
|
84
|
+
* tenants and admin paths.
|
|
85
|
+
*/
|
|
86
|
+
searchType?: SearchType$1;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Wraps a storage engine as an `AnalysisQuerySource` with SQL execution.
|
|
90
|
+
* `queryRows` runs typed builder-state queries; `executeSql` delegates to
|
|
91
|
+
* `engine.runSQL` and requires `opts.fileSets` (with a `FILES` entry so the
|
|
92
|
+
* target table can be resolved for partition lookup).
|
|
93
|
+
*/
|
|
94
|
+
declare function createEngineQuerySource(options: EngineQuerySourceOptions): AnalysisQuerySource;
|
|
95
|
+
/**
|
|
96
|
+
* Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
|
|
97
|
+
* Equivalent to wrapping `createEngineQuerySource`, with omitted searchType
|
|
98
|
+
* defaulted to web at this public helper boundary.
|
|
99
|
+
*/
|
|
100
|
+
declare function runAnalyzerWithEngine(deps: {
|
|
101
|
+
engine: StorageEngine;
|
|
102
|
+
}, ctx: TenantCtx, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
|
|
103
|
+
declare function queryRows<TRow = QueryRow>(source: AnalysisQuerySource, state: BuilderState): Promise<TRow[]>;
|
|
3
104
|
export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
|
package/dist/source/index.mjs
CHANGED
|
@@ -1,2 +1,143 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { engineErrors } from "../errors.mjs";
|
|
2
|
+
import "../_chunks/layout.mjs";
|
|
3
|
+
import { coerceRows } from "../_chunks/coerce.mjs";
|
|
4
|
+
import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "../_chunks/resolver.mjs";
|
|
5
|
+
import { runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
|
|
6
|
+
var AttachedTableMissingError = class extends Error {
|
|
7
|
+
missing;
|
|
8
|
+
engineError;
|
|
9
|
+
constructor(missing) {
|
|
10
|
+
const engineError = engineErrors.attachedTableMissing(missing);
|
|
11
|
+
super(engineError.message);
|
|
12
|
+
this.missing = missing;
|
|
13
|
+
this.name = "AttachedTableMissingError";
|
|
14
|
+
this.engineError = engineError;
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
const ATTACHED_TABLE_CAPABILITIES = {
|
|
18
|
+
fileSets: true,
|
|
19
|
+
attachedTables: true,
|
|
20
|
+
regex: true
|
|
21
|
+
};
|
|
22
|
+
const ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER = {
|
|
23
|
+
...ATTACHED_TABLE_CAPABILITIES,
|
|
24
|
+
adapter: true
|
|
25
|
+
};
|
|
26
|
+
function rewriteForTableSource(sql, schema, fileSets) {
|
|
27
|
+
let out = sql;
|
|
28
|
+
for (const [key, fs] of Object.entries(fileSets)) {
|
|
29
|
+
const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${key}\\}\\}\\s*,\\s*union_by_name\\s*=\\s*true\\s*\\)`, "g");
|
|
30
|
+
out = out.replace(pattern, `${schema}.${fs.table}`);
|
|
31
|
+
}
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
function createAttachedTableSource(runner, options) {
|
|
35
|
+
const { schema, signal, attachedTables, adapter } = options;
|
|
36
|
+
const attachedSet = attachedTables ? new Set(attachedTables) : null;
|
|
37
|
+
return {
|
|
38
|
+
name: "attached-table",
|
|
39
|
+
kind: "browser",
|
|
40
|
+
capabilities: adapter ? ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER : ATTACHED_TABLE_CAPABILITIES,
|
|
41
|
+
adapter,
|
|
42
|
+
async queryRows() {
|
|
43
|
+
throw new Error("attached-table source: queryRows is not supported; use SQL analyzers");
|
|
44
|
+
},
|
|
45
|
+
async executeSql(sql, params, opts) {
|
|
46
|
+
signal?.throwIfAborted();
|
|
47
|
+
const fileSets = opts?.fileSets ?? {};
|
|
48
|
+
if (attachedSet) {
|
|
49
|
+
const missing = [];
|
|
50
|
+
for (const fs of Object.values(fileSets)) if (!attachedSet.has(fs.table)) missing.push(fs.table);
|
|
51
|
+
if (missing.length > 0) throw new AttachedTableMissingError(missing);
|
|
52
|
+
}
|
|
53
|
+
const rewritten = rewriteForTableSource(sql, schema, fileSets);
|
|
54
|
+
return coerceRows(await runner.query(rewritten, params ?? [], signal));
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function createSqlQuerySource(options) {
|
|
59
|
+
const { name, kind, adapter, execute, siteId, searchType, extraCapabilities } = options;
|
|
60
|
+
return {
|
|
61
|
+
name,
|
|
62
|
+
kind,
|
|
63
|
+
capabilities: {
|
|
64
|
+
...adapter.capabilities,
|
|
65
|
+
...extraCapabilities,
|
|
66
|
+
adapter: true
|
|
67
|
+
},
|
|
68
|
+
adapter,
|
|
69
|
+
siteId,
|
|
70
|
+
async queryRows(state) {
|
|
71
|
+
const resolved = resolveToSQL(state, {
|
|
72
|
+
adapter,
|
|
73
|
+
siteId,
|
|
74
|
+
searchType
|
|
75
|
+
});
|
|
76
|
+
return coerceRows(await execute(resolved.sql, resolved.params));
|
|
77
|
+
},
|
|
78
|
+
async executeSql(sql, params) {
|
|
79
|
+
return coerceRows(await execute(sql, params ?? []));
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function isMetricDimension(dim) {
|
|
84
|
+
return [
|
|
85
|
+
"clicks",
|
|
86
|
+
"impressions",
|
|
87
|
+
"ctr",
|
|
88
|
+
"position"
|
|
89
|
+
].includes(dim);
|
|
90
|
+
}
|
|
91
|
+
const ENGINE_QUERY_CAPABILITIES = {
|
|
92
|
+
regex: true,
|
|
93
|
+
multiDataset: false,
|
|
94
|
+
comparisonJoin: false,
|
|
95
|
+
windowTotals: false
|
|
96
|
+
};
|
|
97
|
+
const ENGINE_SOURCE_CAPABILITIES = {
|
|
98
|
+
...ENGINE_QUERY_CAPABILITIES,
|
|
99
|
+
fileSets: true,
|
|
100
|
+
adapter: true
|
|
101
|
+
};
|
|
102
|
+
function createEngineQuerySource(options) {
|
|
103
|
+
const { engine, ctx, searchType } = options;
|
|
104
|
+
return {
|
|
105
|
+
name: "engine",
|
|
106
|
+
kind: "local",
|
|
107
|
+
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
108
|
+
adapter: pgResolverAdapter,
|
|
109
|
+
async queryRows(state) {
|
|
110
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
111
|
+
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
112
|
+
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
113
|
+
return coerceRows((await engine.query({
|
|
114
|
+
...ctx,
|
|
115
|
+
...searchType !== void 0 ? { searchType } : {}
|
|
116
|
+
}, state)).rows);
|
|
117
|
+
},
|
|
118
|
+
async executeSql(sql, params, opts) {
|
|
119
|
+
const fileSets = opts?.fileSets;
|
|
120
|
+
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
121
|
+
const { rows } = await engine.runSQL({
|
|
122
|
+
ctx,
|
|
123
|
+
table: fileSets.FILES.table,
|
|
124
|
+
fileSets,
|
|
125
|
+
sql,
|
|
126
|
+
params: params ?? [],
|
|
127
|
+
...searchType !== void 0 ? { searchType } : {}
|
|
128
|
+
});
|
|
129
|
+
return coerceRows(rows);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
134
|
+
return runAnalyzerFromSource(createEngineQuerySource({
|
|
135
|
+
engine: deps.engine,
|
|
136
|
+
ctx,
|
|
137
|
+
searchType: params.searchType ?? "web"
|
|
138
|
+
}), params, registry);
|
|
139
|
+
}
|
|
140
|
+
async function queryRows(source, state) {
|
|
141
|
+
return await source.queryRows(state);
|
|
142
|
+
}
|
|
2
143
|
export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.3",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -51,6 +51,11 @@
|
|
|
51
51
|
"import": "./dist/ingest.mjs",
|
|
52
52
|
"default": "./dist/ingest.mjs"
|
|
53
53
|
},
|
|
54
|
+
"./ingest-accumulator": {
|
|
55
|
+
"types": "./dist/ingest-accumulator.d.mts",
|
|
56
|
+
"import": "./dist/ingest-accumulator.mjs",
|
|
57
|
+
"default": "./dist/ingest-accumulator.mjs"
|
|
58
|
+
},
|
|
54
59
|
"./sink-node": {
|
|
55
60
|
"types": "./dist/sink-node.d.mts",
|
|
56
61
|
"import": "./dist/sink-node.mjs",
|
|
@@ -71,6 +76,11 @@
|
|
|
71
76
|
"import": "./dist/entities.mjs",
|
|
72
77
|
"default": "./dist/entities.mjs"
|
|
73
78
|
},
|
|
79
|
+
"./entity-keys": {
|
|
80
|
+
"types": "./dist/entity-keys.d.mts",
|
|
81
|
+
"import": "./dist/entity-keys.mjs",
|
|
82
|
+
"default": "./dist/entity-keys.mjs"
|
|
83
|
+
},
|
|
74
84
|
"./rollups": {
|
|
75
85
|
"types": "./dist/rollups.d.mts",
|
|
76
86
|
"import": "./dist/rollups.mjs",
|
|
@@ -166,9 +176,9 @@
|
|
|
166
176
|
"dependencies": {
|
|
167
177
|
"drizzle-orm": "1.0.0-rc.3",
|
|
168
178
|
"proper-lockfile": "^4.1.2",
|
|
169
|
-
"@gscdump/contracts": "^1.0.
|
|
170
|
-
"gscdump": "^1.0.
|
|
171
|
-
"@gscdump/lakehouse": "^1.0.
|
|
179
|
+
"@gscdump/contracts": "^1.0.3",
|
|
180
|
+
"gscdump": "^1.0.3",
|
|
181
|
+
"@gscdump/lakehouse": "^1.0.3"
|
|
172
182
|
},
|
|
173
183
|
"devDependencies": {
|
|
174
184
|
"@duckdb/duckdb-wasm": "1.33.1-dev57.0",
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { ParquetCodec, QueryExecutor, Row } from "./storage.mjs";
|
|
2
|
-
interface DuckDBHandle {
|
|
3
|
-
query: (sql: string, params?: unknown[]) => Promise<Row[]>;
|
|
4
|
-
registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
|
|
5
|
-
copyFileToBuffer: (name: string) => Promise<Uint8Array>;
|
|
6
|
-
dropFiles: (names: string[]) => Promise<void>;
|
|
7
|
-
/**
|
|
8
|
-
* Returns a unique path suitable for `COPY TO '…'` + `copyFileToBuffer`.
|
|
9
|
-
* In Node this is an absolute path under `os.tmpdir()` so DuckDB doesn't
|
|
10
|
-
* litter the CWD; in browsers/Workers it's a plain virtual-FS name.
|
|
11
|
-
*/
|
|
12
|
-
makeTempPath: (ext: string) => string;
|
|
13
|
-
}
|
|
14
|
-
interface DuckDBFactory {
|
|
15
|
-
getDuckDB: () => Promise<DuckDBHandle>;
|
|
16
|
-
}
|
|
17
|
-
interface DuckDBReadOptions {
|
|
18
|
-
/**
|
|
19
|
-
* Maximum non-URI parquet reads held in flight while registering DuckDB
|
|
20
|
-
* buffers. Defaults to 16; callers handling unusually large files can lower
|
|
21
|
-
* it to trade latency for peak JS memory.
|
|
22
|
-
*/
|
|
23
|
-
bufferReadConcurrency?: number;
|
|
24
|
-
}
|
|
25
|
-
declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
|
|
26
|
-
declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
|
|
27
|
-
export { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor };
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import { AnalysisParams } from "./analysis-types.mjs";
|
|
2
|
-
type WindowPreset = 'last-7d' | 'last-28d' | 'last-30d' | 'last-90d' | 'last-180d' | 'last-365d' | 'mtd' | 'ytd' | 'custom';
|
|
3
|
-
type ComparisonMode = 'none' | 'prev-period' | 'yoy';
|
|
4
|
-
interface ResolveWindowOptions {
|
|
5
|
-
preset: WindowPreset;
|
|
6
|
-
comparison?: ComparisonMode;
|
|
7
|
-
anchor?: string;
|
|
8
|
-
start?: string;
|
|
9
|
-
end?: string;
|
|
10
|
-
}
|
|
11
|
-
interface ResolvedWindow {
|
|
12
|
-
start: string;
|
|
13
|
-
end: string;
|
|
14
|
-
days: number;
|
|
15
|
-
comparison?: {
|
|
16
|
-
start: string;
|
|
17
|
-
end: string;
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
interface AnalysisPeriod {
|
|
21
|
-
startDate: string;
|
|
22
|
-
endDate: string;
|
|
23
|
-
}
|
|
24
|
-
interface ComparisonPeriod {
|
|
25
|
-
current: AnalysisPeriod;
|
|
26
|
-
previous: AnalysisPeriod;
|
|
27
|
-
}
|
|
28
|
-
declare function defaultEndDate(): string;
|
|
29
|
-
declare function periodOf(params: AnalysisParams): AnalysisPeriod;
|
|
30
|
-
declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
|
|
31
|
-
declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
|
|
32
|
-
interface PadTimeseriesOptions<T> {
|
|
33
|
-
/** ISO date (YYYY-MM-DD), inclusive lower bound. */
|
|
34
|
-
startDate: string;
|
|
35
|
-
/** ISO date (YYYY-MM-DD), inclusive upper bound. */
|
|
36
|
-
endDate: string;
|
|
37
|
-
/**
|
|
38
|
-
* Row to insert for missing dates. Defaults to `{ clicks: 0, impressions: 0, ctr: 0, position: 0 }`.
|
|
39
|
-
* The `date` field is set automatically.
|
|
40
|
-
*/
|
|
41
|
-
fill?: Omit<T, 'date'>;
|
|
42
|
-
/** Row-field that carries the ISO date. Defaults to `date`. */
|
|
43
|
-
dateKey?: string;
|
|
44
|
-
}
|
|
45
|
-
type DateRowShape = Record<string, unknown> & {
|
|
46
|
-
date?: unknown;
|
|
47
|
-
};
|
|
48
|
-
/**
|
|
49
|
-
* Pad rows so every calendar day in `[startDate, endDate]` appears at least
|
|
50
|
-
* once. Existing dates keep all their rows (grouped timeseries safe).
|
|
51
|
-
*/
|
|
52
|
-
declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
|
|
53
|
-
export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import { ResolverAdapter } from "./types.mjs";
|
|
2
|
-
import { TableName } from "@gscdump/contracts";
|
|
3
|
-
type PgTableKey = TableName;
|
|
4
|
-
declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
|
|
5
|
-
/**
|
|
6
|
-
* Parquet-aware variant of {@link pgResolverAdapter}. Identical SQL output
|
|
7
|
-
* except FROM clauses emit `read_parquet({{FILES}}, union_by_name = true) AS
|
|
8
|
-
* "${tk}"`. The runSQL pipeline substitutes `{{FILES}}` with R2 object keys
|
|
9
|
-
* resolved from the manifest. The `AS "${tk}"` alias is mandatory — drizzle
|
|
10
|
-
* compiles `colRef` to table-qualified `"pages"."url"`, which would not
|
|
11
|
-
* resolve against an unaliased FROM.
|
|
12
|
-
*
|
|
13
|
-
* Single-use: build a fresh adapter per query. Cheap (no I/O) and avoids
|
|
14
|
-
* accidental adapter caching that would lock in a stale `{{FILES}}` set.
|
|
15
|
-
*/
|
|
16
|
-
interface ResolverAdapterOptions {
|
|
17
|
-
/**
|
|
18
|
-
* `queryDim` reads canonical from a joined query dimension. `column` is only
|
|
19
|
-
* for derived canonical rollup relations whose primary relation already
|
|
20
|
-
* carries a null-free `query_canonical` output column.
|
|
21
|
-
*/
|
|
22
|
-
queryCanonicalSource?: 'queryDim' | 'column';
|
|
23
|
-
}
|
|
24
|
-
interface R2SqlResolverAdapterOptions extends ResolverAdapterOptions {
|
|
25
|
-
/**
|
|
26
|
-
* R2 SQL string partition equality can undercount on identity partitions;
|
|
27
|
-
* string-encoded catalogs use CONCAT(col, '') in partition predicates. Int
|
|
28
|
-
* catalogs do not need the workaround and keep bare equality for pruning.
|
|
29
|
-
*/
|
|
30
|
-
partitionKeyEncoding?: 'string' | 'int';
|
|
31
|
-
}
|
|
32
|
-
declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
|
|
33
|
-
/**
|
|
34
|
-
* Multi-tenant pg-flavored adapter for the Iceberg / R2 SQL read path.
|
|
35
|
-
* Identical SQL output to `pgResolverAdapter` except WHERE clauses inject
|
|
36
|
-
* `site_id = ?` AND `search_type = ?` automatically when those scopes are
|
|
37
|
-
* passed to `resolveToSQL`. Required for the Iceberg fact tables which are
|
|
38
|
-
* shared across tenants — querying without these predicates would leak
|
|
39
|
-
* cross-tenant data. Single-use: the adapter has no `tableRef` override,
|
|
40
|
-
* so callers must rewrite bare table names to their qualified form (e.g.
|
|
41
|
-
* `${namespace}.pages`) before sending to R2 SQL.
|
|
42
|
-
*/
|
|
43
|
-
declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
|
|
44
|
-
/**
|
|
45
|
-
* R2 SQL adapter for the Iceberg fact tables.
|
|
46
|
-
*
|
|
47
|
-
* It shares the multi-tenant Iceberg schema with `createIcebergResolverAdapter`
|
|
48
|
-
* but models R2 SQL's narrower execution surface: no window-total plans and no
|
|
49
|
-
* comparison joins. Int-partition catalogs are the default and emit bare
|
|
50
|
-
* equality predicates for pruning. Legacy string-partition catalogs must pass
|
|
51
|
-
* `partitionKeyEncoding: 'string'` to emit `CONCAT(partition_col, '') = ?`,
|
|
52
|
-
* working around R2 SQL's partition-string equality undercount while preserving
|
|
53
|
-
* bound params.
|
|
54
|
-
*/
|
|
55
|
-
declare function createR2SqlResolverAdapter(options?: R2SqlResolverAdapterOptions): ResolverAdapter<PgTableKey>;
|
|
56
|
-
export { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter };
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { TableName } from "./storage.mjs";
|
|
2
|
-
import { LogicalQueryPlan } from "gscdump/query/plan";
|
|
3
|
-
import { BuilderState } from "gscdump/query";
|
|
4
|
-
interface ResolvedQuery {
|
|
5
|
-
sql: string;
|
|
6
|
-
params: unknown[];
|
|
7
|
-
partitions: string[];
|
|
8
|
-
table: TableName;
|
|
9
|
-
filesPlaceholder: string;
|
|
10
|
-
}
|
|
11
|
-
declare const FILES_PLACEHOLDER = "{{FILES}}";
|
|
12
|
-
declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
|
|
13
|
-
declare function resolveParquetSQL(state: BuilderState, table?: TableName): ResolvedQuery;
|
|
14
|
-
declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
|
|
15
|
-
export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles };
|
package/dist/_chunks/source.mjs
DELETED
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
import { engineErrors } from "../errors.mjs";
|
|
2
|
-
import "./layout.mjs";
|
|
3
|
-
import { coerceRows } from "./coerce.mjs";
|
|
4
|
-
import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "./resolver.mjs";
|
|
5
|
-
import { runAnalyzerFromSource } from "./dispatch.mjs";
|
|
6
|
-
var AttachedTableMissingError = class extends Error {
|
|
7
|
-
missing;
|
|
8
|
-
engineError;
|
|
9
|
-
constructor(missing) {
|
|
10
|
-
const engineError = engineErrors.attachedTableMissing(missing);
|
|
11
|
-
super(engineError.message);
|
|
12
|
-
this.missing = missing;
|
|
13
|
-
this.name = "AttachedTableMissingError";
|
|
14
|
-
this.engineError = engineError;
|
|
15
|
-
}
|
|
16
|
-
};
|
|
17
|
-
const ATTACHED_TABLE_CAPABILITIES = {
|
|
18
|
-
fileSets: true,
|
|
19
|
-
attachedTables: true,
|
|
20
|
-
regex: true
|
|
21
|
-
};
|
|
22
|
-
const ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER = {
|
|
23
|
-
...ATTACHED_TABLE_CAPABILITIES,
|
|
24
|
-
adapter: true
|
|
25
|
-
};
|
|
26
|
-
function rewriteForTableSource(sql, schema, fileSets) {
|
|
27
|
-
let out = sql;
|
|
28
|
-
for (const [key, fs] of Object.entries(fileSets)) {
|
|
29
|
-
const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${key}\\}\\}\\s*,\\s*union_by_name\\s*=\\s*true\\s*\\)`, "g");
|
|
30
|
-
out = out.replace(pattern, `${schema}.${fs.table}`);
|
|
31
|
-
}
|
|
32
|
-
return out;
|
|
33
|
-
}
|
|
34
|
-
function createAttachedTableSource(runner, options) {
|
|
35
|
-
const { schema, signal, attachedTables, adapter } = options;
|
|
36
|
-
const attachedSet = attachedTables ? new Set(attachedTables) : null;
|
|
37
|
-
return {
|
|
38
|
-
name: "attached-table",
|
|
39
|
-
kind: "browser",
|
|
40
|
-
capabilities: adapter ? ATTACHED_TABLE_CAPABILITIES_WITH_ADAPTER : ATTACHED_TABLE_CAPABILITIES,
|
|
41
|
-
adapter,
|
|
42
|
-
async queryRows() {
|
|
43
|
-
throw new Error("attached-table source: queryRows is not supported; use SQL analyzers");
|
|
44
|
-
},
|
|
45
|
-
async executeSql(sql, params, opts) {
|
|
46
|
-
signal?.throwIfAborted();
|
|
47
|
-
const fileSets = opts?.fileSets ?? {};
|
|
48
|
-
if (attachedSet) {
|
|
49
|
-
const missing = [];
|
|
50
|
-
for (const fs of Object.values(fileSets)) if (!attachedSet.has(fs.table)) missing.push(fs.table);
|
|
51
|
-
if (missing.length > 0) throw new AttachedTableMissingError(missing);
|
|
52
|
-
}
|
|
53
|
-
const rewritten = rewriteForTableSource(sql, schema, fileSets);
|
|
54
|
-
return coerceRows(await runner.query(rewritten, params ?? [], signal));
|
|
55
|
-
}
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
function createSqlQuerySource(options) {
|
|
59
|
-
const { name, kind, adapter, execute, siteId, searchType, extraCapabilities } = options;
|
|
60
|
-
return {
|
|
61
|
-
name,
|
|
62
|
-
kind,
|
|
63
|
-
capabilities: {
|
|
64
|
-
...adapter.capabilities,
|
|
65
|
-
...extraCapabilities,
|
|
66
|
-
adapter: true
|
|
67
|
-
},
|
|
68
|
-
adapter,
|
|
69
|
-
siteId,
|
|
70
|
-
async queryRows(state) {
|
|
71
|
-
const resolved = resolveToSQL(state, {
|
|
72
|
-
adapter,
|
|
73
|
-
siteId,
|
|
74
|
-
searchType
|
|
75
|
-
});
|
|
76
|
-
return coerceRows(await execute(resolved.sql, resolved.params));
|
|
77
|
-
},
|
|
78
|
-
async executeSql(sql, params) {
|
|
79
|
-
return coerceRows(await execute(sql, params ?? []));
|
|
80
|
-
}
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
function isMetricDimension(dim) {
|
|
84
|
-
return [
|
|
85
|
-
"clicks",
|
|
86
|
-
"impressions",
|
|
87
|
-
"ctr",
|
|
88
|
-
"position"
|
|
89
|
-
].includes(dim);
|
|
90
|
-
}
|
|
91
|
-
const ENGINE_QUERY_CAPABILITIES = {
|
|
92
|
-
regex: true,
|
|
93
|
-
multiDataset: false,
|
|
94
|
-
comparisonJoin: false,
|
|
95
|
-
windowTotals: false
|
|
96
|
-
};
|
|
97
|
-
const ENGINE_SOURCE_CAPABILITIES = {
|
|
98
|
-
...ENGINE_QUERY_CAPABILITIES,
|
|
99
|
-
fileSets: true,
|
|
100
|
-
adapter: true
|
|
101
|
-
};
|
|
102
|
-
function createEngineQuerySource(options) {
|
|
103
|
-
const { engine, ctx, searchType } = options;
|
|
104
|
-
return {
|
|
105
|
-
name: "engine",
|
|
106
|
-
kind: "local",
|
|
107
|
-
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
108
|
-
adapter: pgResolverAdapter,
|
|
109
|
-
async queryRows(state) {
|
|
110
|
-
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
111
|
-
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
112
|
-
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
113
|
-
return coerceRows((await engine.query({
|
|
114
|
-
...ctx,
|
|
115
|
-
...searchType !== void 0 ? { searchType } : {}
|
|
116
|
-
}, state)).rows);
|
|
117
|
-
},
|
|
118
|
-
async executeSql(sql, params, opts) {
|
|
119
|
-
const fileSets = opts?.fileSets;
|
|
120
|
-
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
121
|
-
const { rows } = await engine.runSQL({
|
|
122
|
-
ctx,
|
|
123
|
-
table: fileSets.FILES.table,
|
|
124
|
-
fileSets,
|
|
125
|
-
sql,
|
|
126
|
-
params: params ?? [],
|
|
127
|
-
...searchType !== void 0 ? { searchType } : {}
|
|
128
|
-
});
|
|
129
|
-
return coerceRows(rows);
|
|
130
|
-
}
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
134
|
-
return runAnalyzerFromSource(createEngineQuerySource({
|
|
135
|
-
engine: deps.engine,
|
|
136
|
-
ctx,
|
|
137
|
-
searchType: params.searchType ?? "web"
|
|
138
|
-
}), params, registry);
|
|
139
|
-
}
|
|
140
|
-
async function queryRows(source, state) {
|
|
141
|
-
return await source.queryRows(state);
|
|
142
|
-
}
|
|
143
|
-
export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import { DuckDBHandle } from "../_chunks/duckdb.mjs";
|
|
2
|
-
interface NodeDuckDBOptions {
|
|
3
|
-
verbose?: boolean;
|
|
4
|
-
}
|
|
5
|
-
declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
|
|
6
|
-
declare function resetNodeDuckDB(): void;
|
|
7
|
-
export { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB };
|