@gscdump/engine 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/_chunks/analysis-types.d.mts +47 -0
- package/dist/_chunks/contracts.d.mts +1 -0
- package/dist/_chunks/dispatch.mjs +75 -0
- package/dist/_chunks/registry.d.mts +92 -0
- package/dist/_chunks/resolver.mjs +91 -0
- package/dist/_chunks/source-types.d.mts +31 -0
- package/dist/analysis-types.d.mts +2 -0
- package/dist/analysis-types.mjs +7 -0
- package/dist/analyzer/index.d.mts +59 -0
- package/dist/analyzer/index.mjs +104 -0
- package/dist/contracts.d.mts +1 -1
- package/dist/period/index.d.mts +57 -0
- package/dist/period/index.mjs +150 -0
- package/dist/resolver/index.d.mts +1 -27
- package/dist/resolver/index.mjs +1 -89
- package/dist/scope.d.mts +3 -3
- package/dist/source/index.d.mts +78 -0
- package/dist/source/index.mjs +113 -0
- package/package.json +64 -27
- package/dist/rollups.d.mts +0 -162
- package/dist/rollups.mjs +0 -346
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { MS_PER_DAY, daysAgo, toIsoDate } from "gscdump";
|
|
2
|
+
function defaultEndDate() {
|
|
3
|
+
return daysAgo(3);
|
|
4
|
+
}
|
|
5
|
+
function defaultStartDate() {
|
|
6
|
+
return daysAgo(31);
|
|
7
|
+
}
|
|
8
|
+
function periodOf(params) {
|
|
9
|
+
return {
|
|
10
|
+
startDate: params.startDate || defaultStartDate(),
|
|
11
|
+
endDate: params.endDate || defaultEndDate()
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
function comparisonOf(params) {
|
|
15
|
+
if (!params.prevStartDate || !params.prevEndDate) throw new Error(`${params.type} analysis requires prevStartDate and prevEndDate`);
|
|
16
|
+
return {
|
|
17
|
+
current: periodOf(params),
|
|
18
|
+
previous: {
|
|
19
|
+
startDate: params.prevStartDate,
|
|
20
|
+
endDate: params.prevEndDate
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function parseIso(s) {
|
|
25
|
+
return /* @__PURE__ */ new Date(`${s}T00:00:00Z`);
|
|
26
|
+
}
|
|
27
|
+
function addDays(d, n) {
|
|
28
|
+
return new Date(d.getTime() + n * MS_PER_DAY);
|
|
29
|
+
}
|
|
30
|
+
function daysBetween(start, end) {
|
|
31
|
+
return Math.round((parseIso(end).getTime() - parseIso(start).getTime()) / MS_PER_DAY) + 1;
|
|
32
|
+
}
|
|
33
|
+
function resolveWindow(opts) {
|
|
34
|
+
const anchor = opts.anchor ? parseIso(opts.anchor) : /* @__PURE__ */ new Date();
|
|
35
|
+
const anchorIso = toIsoDate(anchor);
|
|
36
|
+
let start;
|
|
37
|
+
let end;
|
|
38
|
+
switch (opts.preset) {
|
|
39
|
+
case "last-7d":
|
|
40
|
+
end = anchorIso;
|
|
41
|
+
start = toIsoDate(addDays(anchor, -6));
|
|
42
|
+
break;
|
|
43
|
+
case "last-28d":
|
|
44
|
+
end = anchorIso;
|
|
45
|
+
start = toIsoDate(addDays(anchor, -27));
|
|
46
|
+
break;
|
|
47
|
+
case "last-30d":
|
|
48
|
+
end = anchorIso;
|
|
49
|
+
start = toIsoDate(addDays(anchor, -29));
|
|
50
|
+
break;
|
|
51
|
+
case "last-90d":
|
|
52
|
+
end = anchorIso;
|
|
53
|
+
start = toIsoDate(addDays(anchor, -89));
|
|
54
|
+
break;
|
|
55
|
+
case "last-180d":
|
|
56
|
+
end = anchorIso;
|
|
57
|
+
start = toIsoDate(addDays(anchor, -179));
|
|
58
|
+
break;
|
|
59
|
+
case "last-365d":
|
|
60
|
+
end = anchorIso;
|
|
61
|
+
start = toIsoDate(addDays(anchor, -364));
|
|
62
|
+
break;
|
|
63
|
+
case "mtd":
|
|
64
|
+
end = anchorIso;
|
|
65
|
+
start = toIsoDate(new Date(Date.UTC(anchor.getUTCFullYear(), anchor.getUTCMonth(), 1)));
|
|
66
|
+
break;
|
|
67
|
+
case "ytd":
|
|
68
|
+
end = anchorIso;
|
|
69
|
+
start = toIsoDate(new Date(Date.UTC(anchor.getUTCFullYear(), 0, 1)));
|
|
70
|
+
break;
|
|
71
|
+
case "custom":
|
|
72
|
+
if (!opts.start || !opts.end) throw new Error("resolveWindow: preset=custom requires start and end");
|
|
73
|
+
start = opts.start;
|
|
74
|
+
end = opts.end;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
const days = daysBetween(start, end);
|
|
78
|
+
const result = {
|
|
79
|
+
start,
|
|
80
|
+
end,
|
|
81
|
+
days
|
|
82
|
+
};
|
|
83
|
+
const mode = opts.comparison ?? "none";
|
|
84
|
+
if (mode === "prev-period") {
|
|
85
|
+
const prevEnd = toIsoDate(addDays(parseIso(start), -1));
|
|
86
|
+
result.comparison = {
|
|
87
|
+
start: toIsoDate(addDays(parseIso(prevEnd), -(days - 1))),
|
|
88
|
+
end: prevEnd
|
|
89
|
+
};
|
|
90
|
+
} else if (mode === "yoy") {
|
|
91
|
+
const prevEnd = toIsoDate(addDays(parseIso(end), -365));
|
|
92
|
+
result.comparison = {
|
|
93
|
+
start: toIsoDate(addDays(parseIso(start), -365)),
|
|
94
|
+
end: prevEnd
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
function windowToPeriod(w) {
|
|
100
|
+
return {
|
|
101
|
+
startDate: w.start,
|
|
102
|
+
endDate: w.end
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function windowToComparisonPeriod(w) {
|
|
106
|
+
if (!w.comparison) return void 0;
|
|
107
|
+
return {
|
|
108
|
+
current: {
|
|
109
|
+
startDate: w.start,
|
|
110
|
+
endDate: w.end
|
|
111
|
+
},
|
|
112
|
+
previous: {
|
|
113
|
+
startDate: w.comparison.start,
|
|
114
|
+
endDate: w.comparison.end
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
const DEFAULT_FILL = {
|
|
119
|
+
clicks: 0,
|
|
120
|
+
impressions: 0,
|
|
121
|
+
ctr: 0,
|
|
122
|
+
position: 0
|
|
123
|
+
};
|
|
124
|
+
function padTimeseries(rows, options) {
|
|
125
|
+
const { startDate, endDate } = options;
|
|
126
|
+
const dateKey = options.dateKey ?? "date";
|
|
127
|
+
const fill = options.fill ?? DEFAULT_FILL;
|
|
128
|
+
const byDate = /* @__PURE__ */ new Map();
|
|
129
|
+
for (const row of rows) {
|
|
130
|
+
const d = String(row[dateKey]);
|
|
131
|
+
const bucket = byDate.get(d);
|
|
132
|
+
if (bucket) bucket.push(row);
|
|
133
|
+
else byDate.set(d, [row]);
|
|
134
|
+
}
|
|
135
|
+
const result = [];
|
|
136
|
+
const start = /* @__PURE__ */ new Date(`${startDate}T00:00:00Z`);
|
|
137
|
+
const end = /* @__PURE__ */ new Date(`${endDate}T00:00:00Z`);
|
|
138
|
+
if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) throw new Error(`padTimeseries: invalid date range ${startDate}..${endDate}`);
|
|
139
|
+
for (let cursorMs = start.getTime(), endMs = end.getTime(); cursorMs <= endMs; cursorMs += MS_PER_DAY) {
|
|
140
|
+
const dateStr = toIsoDate(new Date(cursorMs));
|
|
141
|
+
const existing = byDate.get(dateStr);
|
|
142
|
+
if (existing) result.push(...existing);
|
|
143
|
+
else result.push({
|
|
144
|
+
...fill,
|
|
145
|
+
[dateKey]: dateStr
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
149
|
+
}
|
|
150
|
+
export { comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { a as ResolvedSQLOptimized, i as ResolvedSQL, n as ExtraQuery, o as ResolverAdapter, r as ResolvedComparisonSQL, s as ResolverOptions, t as ComparisonFilter } from "../_chunks/types.mjs";
|
|
2
|
+
import { a as RowQuerySource, c as isSqlQuerySource, i as QueryRow, n as ExecuteSqlOptions, o as SourceCapabilities, r as FileSet, s as SqlQuerySource, t as AnalysisQuerySource } from "../_chunks/source-types.mjs";
|
|
2
3
|
import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities } from "gscdump/query/plan";
|
|
3
4
|
import { SQL } from "drizzle-orm";
|
|
4
5
|
import { TableName } from "gscdump/contracts";
|
|
@@ -73,33 +74,6 @@ declare function mergeExtras(rows: Record<string, unknown>[], extrasResults: {
|
|
|
73
74
|
key: string;
|
|
74
75
|
results: Record<string, unknown>[];
|
|
75
76
|
}[]): Record<string, unknown>[];
|
|
76
|
-
type QueryRow = Record<string, unknown>;
|
|
77
|
-
interface FileSet {
|
|
78
|
-
table: TableName;
|
|
79
|
-
partitions: string[];
|
|
80
|
-
}
|
|
81
|
-
interface ExecuteSqlOptions {
|
|
82
|
-
fileSets?: Record<string, FileSet>;
|
|
83
|
-
}
|
|
84
|
-
interface SourceCapabilities extends PlannerCapabilities {
|
|
85
|
-
attachedTables?: boolean;
|
|
86
|
-
fileSets?: boolean;
|
|
87
|
-
localSource?: boolean;
|
|
88
|
-
}
|
|
89
|
-
interface RowQuerySource {
|
|
90
|
-
name?: string;
|
|
91
|
-
capabilities: SourceCapabilities;
|
|
92
|
-
queryRows: (state: BuilderState) => Promise<QueryRow[]>;
|
|
93
|
-
readonly executeSql?: undefined;
|
|
94
|
-
}
|
|
95
|
-
interface SqlQuerySource {
|
|
96
|
-
name?: string;
|
|
97
|
-
capabilities: SourceCapabilities;
|
|
98
|
-
queryRows: (state: BuilderState) => Promise<QueryRow[]>;
|
|
99
|
-
executeSql: (sql: string, params?: unknown[], opts?: ExecuteSqlOptions) => Promise<QueryRow[]>;
|
|
100
|
-
}
|
|
101
|
-
type AnalysisQuerySource = RowQuerySource | SqlQuerySource;
|
|
102
|
-
declare function isSqlQuerySource(s: AnalysisQuerySource): s is SqlQuerySource;
|
|
103
77
|
interface CreateSqlQuerySourceOptions<TKey extends string> {
|
|
104
78
|
/** Debug-only identifier surfaced on the source for error messages. */
|
|
105
79
|
name: string;
|
package/dist/resolver/index.mjs
CHANGED
|
@@ -1,91 +1,3 @@
|
|
|
1
|
-
import { t as SCHEMAS } from "../_chunks/schema.mjs";
|
|
2
1
|
import { _ as resolveToSQL, a as createResolverAdapter, c as LOGICAL_DATASETS, d as inferLogicalDataset, f as supportsDimensionOnSurface, g as resolveComparisonSQL, h as mergeExtras, i as compileSqlite, l as assertDimensionsSupported, m as buildTotalsSql, n as pgResolverAdapter, o as createSqlFragments, p as buildExtrasQueries, r as compilePg, s as DIMENSION_SURFACES, t as createParquetResolverAdapter, u as dimensionColumn, v as resolveToSQLOptimized } from "../_chunks/pg-adapter.mjs";
|
|
3
|
-
import {
|
|
4
|
-
function createSqlQuerySource(options) {
|
|
5
|
-
const { name, adapter, execute, siteId, extraCapabilities } = options;
|
|
6
|
-
return {
|
|
7
|
-
name,
|
|
8
|
-
capabilities: {
|
|
9
|
-
...adapter.capabilities,
|
|
10
|
-
...extraCapabilities
|
|
11
|
-
},
|
|
12
|
-
async queryRows(state) {
|
|
13
|
-
const resolved = resolveToSQL(state, {
|
|
14
|
-
adapter,
|
|
15
|
-
siteId
|
|
16
|
-
});
|
|
17
|
-
return execute(resolved.sql, resolved.params);
|
|
18
|
-
},
|
|
19
|
-
executeSql(sql, params) {
|
|
20
|
-
return execute(sql, params ?? []);
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
function collectInternalFilters(filter) {
|
|
25
|
-
if (!filter || !("_filters" in filter)) return [];
|
|
26
|
-
const flat = filter._filters;
|
|
27
|
-
const nested = filter._nestedGroups?.flatMap((group) => collectInternalFilters(group)) ?? [];
|
|
28
|
-
return [...flat, ...nested];
|
|
29
|
-
}
|
|
30
|
-
function getInternalFilters(filter) {
|
|
31
|
-
return collectInternalFilters(filter);
|
|
32
|
-
}
|
|
33
|
-
function getDimensionFilters(filter, isMetricDimension) {
|
|
34
|
-
return collectInternalFilters(filter).filter((f) => f.dimension !== "date" && !isMetricDimension(f.dimension) && f.operator !== "topLevel" && !f.operator.startsWith("metric"));
|
|
35
|
-
}
|
|
36
|
-
function getFilterDimensions(filter, isMetricDimension) {
|
|
37
|
-
return getDimensionFilters(filter, isMetricDimension).map((f) => f.dimension);
|
|
38
|
-
}
|
|
39
|
-
function metricValue(row, metric) {
|
|
40
|
-
const value = row[metric];
|
|
41
|
-
if (typeof value === "number") return value;
|
|
42
|
-
if (typeof value === "bigint") return Number(value);
|
|
43
|
-
if (value == null) return 0;
|
|
44
|
-
return Number(value);
|
|
45
|
-
}
|
|
46
|
-
function dimensionValue(row, dimension) {
|
|
47
|
-
const value = row[dimension];
|
|
48
|
-
return value == null ? "" : String(value);
|
|
49
|
-
}
|
|
50
|
-
function matchesDimensionFilter(row, filter) {
|
|
51
|
-
const raw = dimensionValue(row, filter.dimension);
|
|
52
|
-
const value = filter.dimension === "page" ? normalizeUrl(raw) : raw;
|
|
53
|
-
switch (filter.operator) {
|
|
54
|
-
case "equals": return value === filter.expression;
|
|
55
|
-
case "notEquals": return value !== filter.expression;
|
|
56
|
-
case "contains": return raw.includes(filter.expression);
|
|
57
|
-
case "notContains": return !raw.includes(filter.expression);
|
|
58
|
-
case "includingRegex": return new RegExp(filter.expression).test(raw);
|
|
59
|
-
case "excludingRegex": return !new RegExp(filter.expression).test(raw);
|
|
60
|
-
default: return true;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
function matchesMetricFilter(row, filter) {
|
|
64
|
-
const value = metricValue(row, filter.dimension);
|
|
65
|
-
const target = Number(filter.expression);
|
|
66
|
-
switch (filter.operator) {
|
|
67
|
-
case "metricGte": return value >= target;
|
|
68
|
-
case "metricGt": return value > target;
|
|
69
|
-
case "metricLte": return value <= target;
|
|
70
|
-
case "metricLt": return value < target;
|
|
71
|
-
case "metricBetween": return value >= target && value <= Number(filter.expression2);
|
|
72
|
-
default: return true;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
function matchesTopLevelPage(row) {
|
|
76
|
-
return (normalizeUrl(dimensionValue(row, "page")).match(/\//g)?.length ?? 0) <= 1;
|
|
77
|
-
}
|
|
78
|
-
function assertSchemaInSync(options) {
|
|
79
|
-
const { label, schema, tableKeyToName, mode } = options;
|
|
80
|
-
for (const [key, table] of Object.entries(schema)) {
|
|
81
|
-
const sourceCols = SCHEMAS[tableKeyToName(key)].columns.map((c) => c.name).sort();
|
|
82
|
-
const drizzleCols = Object.keys(table[Symbol.for("drizzle:Columns")] ?? {}).sort();
|
|
83
|
-
const missing = sourceCols.filter((c) => !drizzleCols.includes(c));
|
|
84
|
-
const extra = mode === "exact" ? drizzleCols.filter((c) => !sourceCols.includes(c)) : [];
|
|
85
|
-
if (missing.length > 0 || extra.length > 0) throw new Error(`${label} drizzle schema for '${key}' drifted from SCHEMAS. Missing: [${missing.join(", ")}]. Extra: [${extra.join(", ")}].`);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
function isSqlQuerySource(s) {
|
|
89
|
-
return typeof s.executeSql === "function";
|
|
90
|
-
}
|
|
2
|
+
import { a as getFilterDimensions, c as matchesMetricFilter, d as createSqlQuerySource, i as getDimensionFilters, l as matchesTopLevelPage, n as assertSchemaInSync, o as getInternalFilters, r as dimensionValue, s as matchesDimensionFilter, t as isSqlQuerySource, u as metricValue } from "../_chunks/resolver.mjs";
|
|
91
3
|
export { DIMENSION_SURFACES, LOGICAL_DATASETS, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, compilePg, compileSqlite, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, createSqlQuerySource, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isSqlQuerySource, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, supportsDimensionOnSurface };
|
package/dist/scope.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { SQL } from "drizzle-orm";
|
|
2
2
|
/**
|
|
3
|
-
* Structural subset of `ResolvedWindow` from `@gscdump/
|
|
4
|
-
* Inlined here
|
|
3
|
+
* Structural subset of `ResolvedWindow` from `@gscdump/engine/period`.
|
|
4
|
+
* Inlined here to avoid the cross-module type import in this leaf module;
|
|
5
5
|
* any object with `start`/`end` strings (and optional `days`) satisfies it.
|
|
6
6
|
*/
|
|
7
7
|
interface ResolvedWindow {
|
|
@@ -26,7 +26,7 @@ declare function buildTableScope(table: Record<string, any>, opts: ScopedRunnerO
|
|
|
26
26
|
declare function mergeScope(scope: TableScope, ...extra: SQL[]): SQL | undefined;
|
|
27
27
|
/**
|
|
28
28
|
* Bind `buildTableScope` + `mergeScope` to a specific drizzle schema. Engine
|
|
29
|
-
* adapters (`engine-sqlite`, `engine-wasm`) call this once at module load and
|
|
29
|
+
* adapters (`engine-sqlite`, `engine-duckdb-wasm`) call this once at module load and
|
|
30
30
|
* re-export the returned `scopeFor` / `mergeScope` so consumers get a typed
|
|
31
31
|
* `keyof Schema` table parameter without each adapter re-implementing the
|
|
32
32
|
* pass-through wrapper.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { D as StorageEngine, P as TenantCtx, w as Row } from "../_chunks/storage.mjs";
|
|
2
|
+
import { n as AnalysisResult, t as AnalysisParams } from "../_chunks/analysis-types.mjs";
|
|
3
|
+
import { i as QueryRow, r as FileSet, s as SqlQuerySource, t as AnalysisQuerySource } from "../_chunks/source-types.mjs";
|
|
4
|
+
import { t as AnalyzerRegistry } from "../_chunks/registry.mjs";
|
|
5
|
+
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
6
|
+
import { BuilderState } from "gscdump/query";
|
|
7
|
+
interface AttachedTableRunner {
|
|
8
|
+
/**
|
|
9
|
+
* Run a query with positional (`?`) bound parameters. Return objects keyed
|
|
10
|
+
* by column name. The runner MUST coerce BIGINT → number and DATE → ISO
|
|
11
|
+
* string (or let the analyzer reducer normalize via `num(v)`/`str(v)`).
|
|
12
|
+
*/
|
|
13
|
+
query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<Row[]>;
|
|
14
|
+
}
|
|
15
|
+
interface AttachedTableSourceOptions {
|
|
16
|
+
/** Schema name the exported DuckDB file was attached under — e.g. `gsc`. */
|
|
17
|
+
schema: string;
|
|
18
|
+
/**
|
|
19
|
+
* Abort in-flight queries when the caller no longer cares about the
|
|
20
|
+
* result. Every `runner.query` call receives the same signal.
|
|
21
|
+
*/
|
|
22
|
+
signal?: AbortSignal;
|
|
23
|
+
/**
|
|
24
|
+
* List of table names actually attached to this connection. When provided,
|
|
25
|
+
* `executeSql` short-circuits with a specific "table not attached" error
|
|
26
|
+
* if the SQL plan references a table that isn't in this list — letting
|
|
27
|
+
* callers (e.g. the analytics layer) route to cloud fallback without
|
|
28
|
+
* paying the SQL execution cost. Omit to disable the check.
|
|
29
|
+
*/
|
|
30
|
+
attachedTables?: readonly string[];
|
|
31
|
+
}
|
|
32
|
+
declare class AttachedTableMissingError extends Error {
|
|
33
|
+
readonly missing: readonly string[];
|
|
34
|
+
constructor(missing: readonly string[]);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Swap `read_parquet({{KEY}}, union_by_name = true)` for `<schema>.<table>`.
|
|
38
|
+
* Tolerates whitespace variation. Preserves the rest of the SQL verbatim.
|
|
39
|
+
*/
|
|
40
|
+
declare function rewriteForTableSource(sql: string, schema: string, fileSets: Record<string, FileSet>): string;
|
|
41
|
+
declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
|
|
42
|
+
/**
|
|
43
|
+
* Capabilities the engine query path honors. Matches what the DuckDB compiler
|
|
44
|
+
* passes to `buildLogicalPlan`: regex pushes down; comparison joins and
|
|
45
|
+
* multi-dataset queries belong to the analyzer dispatcher, not the engine's
|
|
46
|
+
* builder-state query path.
|
|
47
|
+
*/
|
|
48
|
+
declare const ENGINE_QUERY_CAPABILITIES: PlannerCapabilities;
|
|
49
|
+
interface EngineQuerySourceOptions {
|
|
50
|
+
engine: StorageEngine;
|
|
51
|
+
ctx: TenantCtx;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Wraps a storage engine as a `SqlQuerySource`. `queryRows` runs typed
|
|
55
|
+
* builder-state queries; `executeSql` delegates to `engine.runSQL` and
|
|
56
|
+
* requires `opts.fileSets` (with a `FILES` entry so the target table can be
|
|
57
|
+
* resolved for partition lookup).
|
|
58
|
+
*/
|
|
59
|
+
declare function createEngineQuerySource(options: EngineQuerySourceOptions): SqlQuerySource;
|
|
60
|
+
/**
|
|
61
|
+
* Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
|
|
62
|
+
* Equivalent to
|
|
63
|
+
* `runAnalyzerFromSource(createEngineQuerySource({ engine, ctx }), params, registry)`.
|
|
64
|
+
*/
|
|
65
|
+
declare function runAnalyzerWithEngine(deps: {
|
|
66
|
+
engine: StorageEngine;
|
|
67
|
+
}, ctx: TenantCtx, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
|
|
68
|
+
interface TypedQuery<TRow> {
|
|
69
|
+
state: BuilderState;
|
|
70
|
+
readonly __row?: TRow;
|
|
71
|
+
}
|
|
72
|
+
declare function typedQuery<TRow>(state: BuilderState): TypedQuery<TRow>;
|
|
73
|
+
declare function queryRows<TRow = QueryRow>(source: AnalysisQuerySource, query: BuilderState | TypedQuery<TRow>): Promise<TRow[]>;
|
|
74
|
+
declare function queryComparisonRows<TRow = QueryRow>(source: AnalysisQuerySource, current: BuilderState | TypedQuery<TRow>, previous: BuilderState | TypedQuery<TRow>): Promise<{
|
|
75
|
+
current: TRow[];
|
|
76
|
+
previous: TRow[];
|
|
77
|
+
}>;
|
|
78
|
+
export { AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, TypedQuery, createAttachedTableSource, createEngineQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { l as assertDimensionsSupported } from "../_chunks/pg-adapter.mjs";
|
|
2
|
+
import { a as getFilterDimensions } from "../_chunks/resolver.mjs";
|
|
3
|
+
import { n as runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
|
|
4
|
+
var AttachedTableMissingError = class extends Error {
|
|
5
|
+
constructor(missing) {
|
|
6
|
+
super(`attached-table source: required table(s) not attached: ${missing.join(", ")}`);
|
|
7
|
+
this.missing = missing;
|
|
8
|
+
this.name = "AttachedTableMissingError";
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
const ATTACHED_TABLE_CAPABILITIES = {
|
|
12
|
+
fileSets: true,
|
|
13
|
+
attachedTables: true,
|
|
14
|
+
regex: true
|
|
15
|
+
};
|
|
16
|
+
function rewriteForTableSource(sql, schema, fileSets) {
|
|
17
|
+
let out = sql;
|
|
18
|
+
for (const [key, fs] of Object.entries(fileSets)) {
|
|
19
|
+
const pattern = new RegExp(`read_parquet\\(\\s*\\{\\{${key}\\}\\}\\s*,\\s*union_by_name\\s*=\\s*true\\s*\\)`, "g");
|
|
20
|
+
out = out.replace(pattern, `${schema}.${fs.table}`);
|
|
21
|
+
}
|
|
22
|
+
return out;
|
|
23
|
+
}
|
|
24
|
+
function createAttachedTableSource(runner, options) {
|
|
25
|
+
const { schema, signal, attachedTables } = options;
|
|
26
|
+
const attachedSet = attachedTables ? new Set(attachedTables) : null;
|
|
27
|
+
return {
|
|
28
|
+
name: "attached-table",
|
|
29
|
+
capabilities: ATTACHED_TABLE_CAPABILITIES,
|
|
30
|
+
async queryRows() {
|
|
31
|
+
throw new Error("attached-table source: queryRows is not supported; use SQL analyzers");
|
|
32
|
+
},
|
|
33
|
+
async executeSql(sql, params, opts) {
|
|
34
|
+
signal?.throwIfAborted();
|
|
35
|
+
const fileSets = opts?.fileSets ?? {};
|
|
36
|
+
if (attachedSet) {
|
|
37
|
+
const missing = [];
|
|
38
|
+
for (const fs of Object.values(fileSets)) if (!attachedSet.has(fs.table)) missing.push(fs.table);
|
|
39
|
+
if (missing.length > 0) throw new AttachedTableMissingError(missing);
|
|
40
|
+
}
|
|
41
|
+
const rewritten = rewriteForTableSource(sql, schema, fileSets);
|
|
42
|
+
return await runner.query(rewritten, params ?? [], signal);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function isMetricDimension(dim) {
|
|
47
|
+
return [
|
|
48
|
+
"clicks",
|
|
49
|
+
"impressions",
|
|
50
|
+
"ctr",
|
|
51
|
+
"position"
|
|
52
|
+
].includes(dim);
|
|
53
|
+
}
|
|
54
|
+
const ENGINE_QUERY_CAPABILITIES = {
|
|
55
|
+
regex: true,
|
|
56
|
+
multiDataset: false,
|
|
57
|
+
comparisonJoin: false,
|
|
58
|
+
windowTotals: false
|
|
59
|
+
};
|
|
60
|
+
const ENGINE_SOURCE_CAPABILITIES = {
|
|
61
|
+
...ENGINE_QUERY_CAPABILITIES,
|
|
62
|
+
fileSets: true,
|
|
63
|
+
localSource: true
|
|
64
|
+
};
|
|
65
|
+
function createEngineQuerySource(options) {
|
|
66
|
+
const { engine, ctx } = options;
|
|
67
|
+
return {
|
|
68
|
+
name: "engine",
|
|
69
|
+
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
70
|
+
async queryRows(state) {
|
|
71
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
72
|
+
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
73
|
+
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
74
|
+
return (await engine.query(ctx, state)).rows;
|
|
75
|
+
},
|
|
76
|
+
async executeSql(sql, params, opts) {
|
|
77
|
+
const fileSets = opts?.fileSets;
|
|
78
|
+
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
79
|
+
const { rows } = await engine.runSQL({
|
|
80
|
+
ctx,
|
|
81
|
+
table: fileSets.FILES.table,
|
|
82
|
+
fileSets,
|
|
83
|
+
sql,
|
|
84
|
+
params: params ?? []
|
|
85
|
+
});
|
|
86
|
+
return rows;
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
91
|
+
return runAnalyzerFromSource(createEngineQuerySource({
|
|
92
|
+
engine: deps.engine,
|
|
93
|
+
ctx
|
|
94
|
+
}), params, registry);
|
|
95
|
+
}
|
|
96
|
+
function typedQuery(state) {
|
|
97
|
+
return { state };
|
|
98
|
+
}
|
|
99
|
+
function isTypedQuery(value) {
|
|
100
|
+
return "state" in value;
|
|
101
|
+
}
|
|
102
|
+
async function queryRows(source, query) {
|
|
103
|
+
const state = isTypedQuery(query) ? query.state : query;
|
|
104
|
+
return await source.queryRows(state);
|
|
105
|
+
}
|
|
106
|
+
async function queryComparisonRows(source, current, previous) {
|
|
107
|
+
const [currentRows, previousRows] = await Promise.all([queryRows(source, current), queryRows(source, previous)]);
|
|
108
|
+
return {
|
|
109
|
+
current: currentRows,
|
|
110
|
+
previous: previousRows
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.3",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -23,91 +23,128 @@
|
|
|
23
23
|
"exports": {
|
|
24
24
|
".": {
|
|
25
25
|
"types": "./dist/index.d.mts",
|
|
26
|
-
"import": "./dist/index.mjs"
|
|
26
|
+
"import": "./dist/index.mjs",
|
|
27
|
+
"default": "./dist/index.mjs"
|
|
27
28
|
},
|
|
28
29
|
"./contracts": {
|
|
29
30
|
"types": "./dist/contracts.d.mts",
|
|
30
|
-
"import": "./dist/contracts.mjs"
|
|
31
|
+
"import": "./dist/contracts.mjs",
|
|
32
|
+
"default": "./dist/contracts.mjs"
|
|
31
33
|
},
|
|
32
34
|
"./snapshot": {
|
|
33
35
|
"types": "./dist/snapshot.d.mts",
|
|
34
|
-
"import": "./dist/snapshot.mjs"
|
|
36
|
+
"import": "./dist/snapshot.mjs",
|
|
37
|
+
"default": "./dist/snapshot.mjs"
|
|
35
38
|
},
|
|
36
39
|
"./planner": {
|
|
37
40
|
"types": "./dist/planner.d.mts",
|
|
38
|
-
"import": "./dist/planner.mjs"
|
|
41
|
+
"import": "./dist/planner.mjs",
|
|
42
|
+
"default": "./dist/planner.mjs"
|
|
39
43
|
},
|
|
40
44
|
"./schema": {
|
|
41
45
|
"types": "./dist/schema.d.mts",
|
|
42
|
-
"import": "./dist/schema.mjs"
|
|
46
|
+
"import": "./dist/schema.mjs",
|
|
47
|
+
"default": "./dist/schema.mjs"
|
|
43
48
|
},
|
|
44
49
|
"./ingest": {
|
|
45
50
|
"types": "./dist/ingest.d.mts",
|
|
46
|
-
"import": "./dist/ingest.mjs"
|
|
51
|
+
"import": "./dist/ingest.mjs",
|
|
52
|
+
"default": "./dist/ingest.mjs"
|
|
47
53
|
},
|
|
48
54
|
"./sql": {
|
|
49
55
|
"types": "./dist/sql-bind.d.mts",
|
|
50
|
-
"import": "./dist/sql-bind.mjs"
|
|
56
|
+
"import": "./dist/sql-bind.mjs",
|
|
57
|
+
"default": "./dist/sql-bind.mjs"
|
|
51
58
|
},
|
|
52
59
|
"./sql-fragments": {
|
|
53
60
|
"types": "./dist/sql-fragments.d.mts",
|
|
54
|
-
"import": "./dist/sql-fragments.mjs"
|
|
55
|
-
|
|
56
|
-
"./rollups": {
|
|
57
|
-
"types": "./dist/rollups.d.mts",
|
|
58
|
-
"import": "./dist/rollups.mjs"
|
|
61
|
+
"import": "./dist/sql-fragments.mjs",
|
|
62
|
+
"default": "./dist/sql-fragments.mjs"
|
|
59
63
|
},
|
|
60
64
|
"./entities": {
|
|
61
65
|
"types": "./dist/entities.d.mts",
|
|
62
|
-
"import": "./dist/entities.mjs"
|
|
66
|
+
"import": "./dist/entities.mjs",
|
|
67
|
+
"default": "./dist/entities.mjs"
|
|
63
68
|
},
|
|
64
69
|
"./node": {
|
|
65
70
|
"types": "./dist/adapters/duckdb-node.d.mts",
|
|
66
|
-
"import": "./dist/adapters/duckdb-node.mjs"
|
|
71
|
+
"import": "./dist/adapters/duckdb-node.mjs",
|
|
72
|
+
"default": "./dist/adapters/duckdb-node.mjs"
|
|
67
73
|
},
|
|
68
74
|
"./node-harness": {
|
|
69
75
|
"types": "./dist/adapters/node-harness.d.mts",
|
|
70
|
-
"import": "./dist/adapters/node-harness.mjs"
|
|
76
|
+
"import": "./dist/adapters/node-harness.mjs",
|
|
77
|
+
"default": "./dist/adapters/node-harness.mjs"
|
|
71
78
|
},
|
|
72
79
|
"./filesystem": {
|
|
73
80
|
"types": "./dist/adapters/filesystem.d.mts",
|
|
74
|
-
"import": "./dist/adapters/filesystem.mjs"
|
|
81
|
+
"import": "./dist/adapters/filesystem.mjs",
|
|
82
|
+
"default": "./dist/adapters/filesystem.mjs"
|
|
75
83
|
},
|
|
76
84
|
"./http": {
|
|
77
85
|
"types": "./dist/adapters/http.d.mts",
|
|
78
|
-
"import": "./dist/adapters/http.mjs"
|
|
86
|
+
"import": "./dist/adapters/http.mjs",
|
|
87
|
+
"default": "./dist/adapters/http.mjs"
|
|
79
88
|
},
|
|
80
89
|
"./hyparquet": {
|
|
81
90
|
"types": "./dist/adapters/hyparquet.d.mts",
|
|
82
|
-
"import": "./dist/adapters/hyparquet.mjs"
|
|
91
|
+
"import": "./dist/adapters/hyparquet.mjs",
|
|
92
|
+
"default": "./dist/adapters/hyparquet.mjs"
|
|
83
93
|
},
|
|
84
94
|
"./r2": {
|
|
85
95
|
"types": "./dist/adapters/r2.d.mts",
|
|
86
|
-
"import": "./dist/adapters/r2.mjs"
|
|
96
|
+
"import": "./dist/adapters/r2.mjs",
|
|
97
|
+
"default": "./dist/adapters/r2.mjs"
|
|
87
98
|
},
|
|
88
99
|
"./r2-manifest": {
|
|
89
100
|
"types": "./dist/adapters/r2-manifest.d.mts",
|
|
90
|
-
"import": "./dist/adapters/r2-manifest.mjs"
|
|
101
|
+
"import": "./dist/adapters/r2-manifest.mjs",
|
|
102
|
+
"default": "./dist/adapters/r2-manifest.mjs"
|
|
91
103
|
},
|
|
92
104
|
"./resolver": {
|
|
93
105
|
"types": "./dist/resolver/index.d.mts",
|
|
94
|
-
"import": "./dist/resolver/index.mjs"
|
|
106
|
+
"import": "./dist/resolver/index.mjs",
|
|
107
|
+
"default": "./dist/resolver/index.mjs"
|
|
108
|
+
},
|
|
109
|
+
"./analyzer": {
|
|
110
|
+
"types": "./dist/analyzer/index.d.mts",
|
|
111
|
+
"import": "./dist/analyzer/index.mjs",
|
|
112
|
+
"default": "./dist/analyzer/index.mjs"
|
|
113
|
+
},
|
|
114
|
+
"./analysis-types": {
|
|
115
|
+
"types": "./dist/analysis-types.d.mts",
|
|
116
|
+
"import": "./dist/analysis-types.mjs",
|
|
117
|
+
"default": "./dist/analysis-types.mjs"
|
|
118
|
+
},
|
|
119
|
+
"./period": {
|
|
120
|
+
"types": "./dist/period/index.d.mts",
|
|
121
|
+
"import": "./dist/period/index.mjs",
|
|
122
|
+
"default": "./dist/period/index.mjs"
|
|
123
|
+
},
|
|
124
|
+
"./source": {
|
|
125
|
+
"types": "./dist/source/index.d.mts",
|
|
126
|
+
"import": "./dist/source/index.mjs",
|
|
127
|
+
"default": "./dist/source/index.mjs"
|
|
95
128
|
},
|
|
96
129
|
"./scope": {
|
|
97
130
|
"types": "./dist/scope.d.mts",
|
|
98
|
-
"import": "./dist/scope.mjs"
|
|
131
|
+
"import": "./dist/scope.mjs",
|
|
132
|
+
"default": "./dist/scope.mjs"
|
|
99
133
|
},
|
|
100
134
|
"./arrow": {
|
|
101
135
|
"types": "./dist/arrow-utils.d.mts",
|
|
102
|
-
"import": "./dist/arrow-utils.mjs"
|
|
136
|
+
"import": "./dist/arrow-utils.mjs",
|
|
137
|
+
"default": "./dist/arrow-utils.mjs"
|
|
103
138
|
},
|
|
104
139
|
"./inspection-sqlite-node": {
|
|
105
140
|
"types": "./dist/adapters/inspection-sqlite-node.d.mts",
|
|
106
|
-
"import": "./dist/adapters/inspection-sqlite-node.mjs"
|
|
141
|
+
"import": "./dist/adapters/inspection-sqlite-node.mjs",
|
|
142
|
+
"default": "./dist/adapters/inspection-sqlite-node.mjs"
|
|
107
143
|
},
|
|
108
144
|
"./inspection-sqlite-browser": {
|
|
109
145
|
"types": "./dist/adapters/inspection-sqlite-browser.d.mts",
|
|
110
|
-
"import": "./dist/adapters/inspection-sqlite-browser.mjs"
|
|
146
|
+
"import": "./dist/adapters/inspection-sqlite-browser.mjs",
|
|
147
|
+
"default": "./dist/adapters/inspection-sqlite-browser.mjs"
|
|
111
148
|
}
|
|
112
149
|
},
|
|
113
150
|
"main": "./dist/index.mjs",
|
|
@@ -145,7 +182,7 @@
|
|
|
145
182
|
"dependencies": {
|
|
146
183
|
"drizzle-orm": "^0.45.2",
|
|
147
184
|
"proper-lockfile": "^4.1.2",
|
|
148
|
-
"gscdump": "0.7.
|
|
185
|
+
"gscdump": "0.7.3"
|
|
149
186
|
},
|
|
150
187
|
"devDependencies": {
|
|
151
188
|
"@duckdb/duckdb-wasm": "^1.32.0",
|