@gscdump/cloudflare 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.mjs +50 -0
- package/dist/engine.d.mts +17 -0
- package/dist/engine.mjs +35 -0
- package/dist/env.d.mts +45 -0
- package/dist/index.d.mts +4 -89
- package/dist/index.mjs +3 -475
- package/dist/inflight-dedupe.d.mts +15 -0
- package/dist/inflight-dedupe.mjs +37 -0
- package/dist/server-tail/archetype-sql.d.mts +38 -0
- package/dist/server-tail/archetype-sql.mjs +345 -0
- package/dist/server-tail/dispatcher.d.mts +47 -0
- package/dist/server-tail/dispatcher.mjs +87 -0
- package/dist/server-tail/duckdb-iceberg-executor.d.mts +88 -0
- package/dist/server-tail/duckdb-iceberg-executor.mjs +77 -0
- package/dist/server-tail/index.d.mts +4 -249
- package/dist/server-tail/index.mjs +4 -660
- package/dist/server-tail/r2-sql-client.d.mts +89 -0
- package/dist/server-tail/r2-sql-client.mjs +159 -0
- package/dist/workers-duckdb.d.mts +19 -0
- package/dist/workers-duckdb.mjs +360 -0
- package/package.json +10 -5
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
const TABLE_PLACEHOLDER = "{{TABLE}}";
|
|
2
|
+
const FACT_ALIAS = "fact";
|
|
3
|
+
function factTableRef() {
|
|
4
|
+
return `${TABLE_PLACEHOLDER} AS ${FACT_ALIAS}`;
|
|
5
|
+
}
|
|
6
|
+
function dimColumn(dim) {
|
|
7
|
+
if (dim === "page") return "url";
|
|
8
|
+
if (dim === "queryCanonical") return `COALESCE((SELECT qd.query_canonical FROM query_dim qd WHERE qd.query = ${FACT_ALIAS}.query LIMIT 1), ${FACT_ALIAS}.query)`;
|
|
9
|
+
return dim;
|
|
10
|
+
}
|
|
11
|
+
function dimSelect(dim) {
|
|
12
|
+
const col = dimColumn(dim);
|
|
13
|
+
return dim === "queryCanonical" ? `${col} AS queryCanonical` : col;
|
|
14
|
+
}
|
|
15
|
+
function tableForDimensions(dims) {
|
|
16
|
+
const set = new Set(dims.filter((d) => d !== "date"));
|
|
17
|
+
if (set.has("page") && (set.has("query") || set.has("queryCanonical"))) return "page_queries";
|
|
18
|
+
if (set.has("query") || set.has("queryCanonical")) return "queries";
|
|
19
|
+
if (set.has("country")) return "countries";
|
|
20
|
+
if (set.has("device")) return "dates";
|
|
21
|
+
return "pages";
|
|
22
|
+
}
|
|
23
|
+
function tableForTopNBreakdown(q) {
|
|
24
|
+
const dims = [q.dimension];
|
|
25
|
+
for (const facet of q.facets ?? []) {
|
|
26
|
+
const hasPage = q.dimension === "page" || facet.column === "page";
|
|
27
|
+
const hasQuery = q.dimension === "query" || q.dimension === "queryCanonical" || facet.column === "query" || facet.column === "queryCanonical";
|
|
28
|
+
if (hasPage && hasQuery) dims.push(facet.column);
|
|
29
|
+
}
|
|
30
|
+
return tableForDimensions(dims);
|
|
31
|
+
}
|
|
32
|
+
function metricExpr(metric) {
|
|
33
|
+
switch (metric) {
|
|
34
|
+
case "clicks": return "SUM(clicks) AS clicks";
|
|
35
|
+
case "impressions": return "SUM(impressions) AS impressions";
|
|
36
|
+
case "ctr": return "SUM(clicks) / NULLIF(SUM(impressions), 0) AS ctr";
|
|
37
|
+
case "position": return "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position";
|
|
38
|
+
default: throw new Error(`[archetype-sql] unknown metric: ${JSON.stringify(metric)}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function metricAlias(metric) {
|
|
42
|
+
switch (metric) {
|
|
43
|
+
case "clicks":
|
|
44
|
+
case "impressions":
|
|
45
|
+
case "ctr":
|
|
46
|
+
case "position": return metric;
|
|
47
|
+
default: throw new Error(`[archetype-sql] unknown order metric: ${JSON.stringify(metric)}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const DEVICE_SUFFIXES = [
|
|
51
|
+
"desktop",
|
|
52
|
+
"mobile",
|
|
53
|
+
"tablet"
|
|
54
|
+
];
|
|
55
|
+
function metricExprForSource(metric, source) {
|
|
56
|
+
switch (metric) {
|
|
57
|
+
case "clicks": return `SUM(${source.clicks}) AS clicks`;
|
|
58
|
+
case "impressions": return `SUM(${source.impressions}) AS impressions`;
|
|
59
|
+
case "ctr": return `SUM(${source.clicks}) / NULLIF(SUM(${source.impressions}), 0) AS ctr`;
|
|
60
|
+
case "position": return `SUM(${source.sumPosition}) / NULLIF(SUM(${source.impressions}), 0) + 1 AS position`;
|
|
61
|
+
default: throw new Error(`[archetype-sql] unknown metric: ${JSON.stringify(metric)}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function deviceSource(suffix) {
|
|
65
|
+
return {
|
|
66
|
+
clicks: `clicks_${suffix}`,
|
|
67
|
+
impressions: `impressions_${suffix}`,
|
|
68
|
+
sumPosition: `sum_position_${suffix}`
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
const STD_METRICS = [
|
|
72
|
+
"clicks",
|
|
73
|
+
"impressions",
|
|
74
|
+
"ctr",
|
|
75
|
+
"position"
|
|
76
|
+
];
|
|
77
|
+
function coalesceMetric(metric, src, alias) {
|
|
78
|
+
const ref = `${src}.${metric}`;
|
|
79
|
+
return metric === "clicks" || metric === "impressions" ? `CAST(COALESCE(${ref}, 0) AS DOUBLE) AS ${alias}` : `COALESCE(${ref}, 0) AS ${alias}`;
|
|
80
|
+
}
|
|
81
|
+
function prevAlias(metric) {
|
|
82
|
+
return `prev${metric.charAt(0).toUpperCase()}${metric.slice(1)}`;
|
|
83
|
+
}
|
|
84
|
+
function moverClause(movers) {
|
|
85
|
+
const curClicks = "COALESCE(c.clicks, 0)";
|
|
86
|
+
const prevClicks = "COALESCE(p.clicks, 0)";
|
|
87
|
+
const curImpr = "COALESCE(c.impressions, 0)";
|
|
88
|
+
const prevImpr = "COALESCE(p.impressions, 0)";
|
|
89
|
+
switch (movers) {
|
|
90
|
+
case "improving": return { where: `${curClicks} > ${prevClicks}` };
|
|
91
|
+
case "declining": return { where: `${curClicks} < ${prevClicks}` };
|
|
92
|
+
case "new": return { where: `${prevImpr} = 0 AND ${curImpr} > 0` };
|
|
93
|
+
case "lost": return { where: `${curImpr} = 0 AND ${prevImpr} > 0` };
|
|
94
|
+
default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function moverOrderByAlias(movers) {
|
|
98
|
+
switch (movers) {
|
|
99
|
+
case "improving": return "(clicks - prevClicks) DESC";
|
|
100
|
+
case "declining": return "(clicks - prevClicks) ASC";
|
|
101
|
+
case "new": return "clicks DESC, impressions DESC";
|
|
102
|
+
case "lost": return "prevImpressions DESC";
|
|
103
|
+
default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function sqlStringLiteral(value) {
|
|
107
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
108
|
+
}
|
|
109
|
+
function partitionWhere(q, partitionPruned = false, mode = "bare") {
|
|
110
|
+
if (partitionPruned) return {
|
|
111
|
+
clause: "date BETWEEN ? AND ?",
|
|
112
|
+
params: [q.range.start, q.range.end]
|
|
113
|
+
};
|
|
114
|
+
return {
|
|
115
|
+
clause: `${mode === "r2-sql-concat" ? "CONCAT(site_id, '')" : "site_id"} = ? AND ${mode === "r2-sql-concat" ? "CONCAT(search_type, '')" : "search_type"} = ? AND date BETWEEN ? AND ?`,
|
|
116
|
+
params: [
|
|
117
|
+
q.siteId,
|
|
118
|
+
q.searchType,
|
|
119
|
+
q.range.start,
|
|
120
|
+
q.range.end
|
|
121
|
+
]
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function facetPredicate(query) {
|
|
125
|
+
const facets = query.facets;
|
|
126
|
+
if (!facets?.length) return {
|
|
127
|
+
sql: "",
|
|
128
|
+
params: []
|
|
129
|
+
};
|
|
130
|
+
const parts = [];
|
|
131
|
+
const params = [];
|
|
132
|
+
for (const f of facets) {
|
|
133
|
+
const col = dimColumn(f.column);
|
|
134
|
+
switch (f.op) {
|
|
135
|
+
case "eq":
|
|
136
|
+
parts.push(`${col} = ?`);
|
|
137
|
+
params.push(f.value);
|
|
138
|
+
break;
|
|
139
|
+
case "regex":
|
|
140
|
+
parts.push(`regexp_matches(LOWER(${col}), ?)`);
|
|
141
|
+
params.push(f.value);
|
|
142
|
+
break;
|
|
143
|
+
case "notRegex":
|
|
144
|
+
parts.push(`NOT regexp_matches(LOWER(${col}), ?)`);
|
|
145
|
+
params.push(f.value);
|
|
146
|
+
break;
|
|
147
|
+
default: throw new Error(`[archetype-sql] unknown facet op: ${f.op}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
sql: parts.length ? ` AND ${parts.join(" AND ")}` : "",
|
|
152
|
+
params
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
function buildSiteDailyTimeseries(q, pruned, mode) {
|
|
156
|
+
const w = partitionWhere(q, pruned, mode);
|
|
157
|
+
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
158
|
+
return {
|
|
159
|
+
table: "dates",
|
|
160
|
+
params: w.params,
|
|
161
|
+
sql: `SELECT date, ${metrics} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date ORDER BY date ASC`
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
function buildEntityDailyTimeseries(q, pruned, mode) {
|
|
165
|
+
const table = tableForDimensions([q.entity.dimension]);
|
|
166
|
+
const w = partitionWhere(q, pruned, mode);
|
|
167
|
+
const col = dimColumn(q.entity.dimension);
|
|
168
|
+
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
169
|
+
return {
|
|
170
|
+
table,
|
|
171
|
+
params: [...w.params, q.entity.value],
|
|
172
|
+
sql: `SELECT date, ${metrics} FROM ${factTableRef()} WHERE ${w.clause} AND ${col} = ? GROUP BY date ORDER BY date ASC`
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
function buildEntityDailySparkline(q, pruned, mode) {
|
|
176
|
+
const table = tableForDimensions([q.dimension]);
|
|
177
|
+
const w = partitionWhere(q, pruned, mode);
|
|
178
|
+
const col = dimColumn(q.dimension);
|
|
179
|
+
if (q.entities.length === 0) throw new Error("entity-daily-sparkline: empty entities - resolver must pre-resolve the top-N list");
|
|
180
|
+
const inList = q.entities.map(sqlStringLiteral).join(", ");
|
|
181
|
+
return {
|
|
182
|
+
table,
|
|
183
|
+
params: w.params,
|
|
184
|
+
sql: `SELECT date, ${col} AS entity, ${metricExpr(q.metric)} FROM ${factTableRef()} WHERE ${w.clause} AND ${col} IN (${inList}) GROUP BY date, ${col} ORDER BY date ASC`
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function buildTopNBreakdown(q, pruned, mode) {
|
|
188
|
+
const table = tableForTopNBreakdown(q);
|
|
189
|
+
const w = partitionWhere(q, pruned, mode);
|
|
190
|
+
if (!q.orderBy || !q.orderBy.metric || !q.orderBy.dir) throw new Error(`[archetype-sql] top-n-breakdown requires orderBy.{metric,dir}, got: ${JSON.stringify(q.orderBy)}`);
|
|
191
|
+
const order = `${metricAlias(q.orderBy.metric)} ${q.orderBy.dir.toUpperCase()}`;
|
|
192
|
+
const limit = `LIMIT ${Math.max(0, Math.floor(q.limit))}`;
|
|
193
|
+
const offset = q.offset && q.offset > 0 ? ` OFFSET ${Math.floor(q.offset)}` : "";
|
|
194
|
+
const metricList0 = q.metrics.includes(q.orderBy.metric) ? q.metrics : [...q.metrics, q.orderBy.metric];
|
|
195
|
+
const metricList = q.compareRange && q.movers ? [.../* @__PURE__ */ new Set([
|
|
196
|
+
...metricList0,
|
|
197
|
+
"clicks",
|
|
198
|
+
"impressions"
|
|
199
|
+
])] : metricList0;
|
|
200
|
+
const variantSel = q.dimension === "queryCanonical" ? ", COUNT(DISTINCT query) AS variantCount" : "";
|
|
201
|
+
if (q.dimension === "device") {
|
|
202
|
+
if (q.compareRange) {
|
|
203
|
+
const wPrev = partitionWhere({
|
|
204
|
+
...q,
|
|
205
|
+
range: q.compareRange
|
|
206
|
+
}, pruned, mode);
|
|
207
|
+
const deviceSelects = (clause, ml) => DEVICE_SUFFIXES.map((suffix) => {
|
|
208
|
+
const source = deviceSource(suffix);
|
|
209
|
+
const metrics = ml.map((m) => metricExprForSource(m, source)).join(", ");
|
|
210
|
+
return `SELECT '${suffix.toUpperCase()}' AS device, ${metrics} FROM ${factTableRef()} WHERE ${clause}`;
|
|
211
|
+
}).join(" UNION ALL ");
|
|
212
|
+
const inner = `SELECT COALESCE(c.device, p.device) AS device, ${metricList.map((m) => coalesceMetric(m, "c", m)).join(", ")}, ${STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ")} FROM cur c FULL OUTER JOIN prev p ON c.device = p.device`;
|
|
213
|
+
const sql = `WITH cur AS (${deviceSelects(w.clause, metricList)}), prev AS (${deviceSelects(wPrev.clause, STD_METRICS)}) SELECT * FROM (${inner}) t ORDER BY ${order} ${limit}${offset}`;
|
|
214
|
+
return {
|
|
215
|
+
table,
|
|
216
|
+
params: [...DEVICE_SUFFIXES.flatMap(() => w.params), ...DEVICE_SUFFIXES.flatMap(() => wPrev.params)],
|
|
217
|
+
sql
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
const sql = `${DEVICE_SUFFIXES.map((suffix) => {
|
|
221
|
+
const source = deviceSource(suffix);
|
|
222
|
+
const metrics = metricList.map((m) => metricExprForSource(m, source)).join(", ");
|
|
223
|
+
return `SELECT '${suffix.toUpperCase()}' AS device, ${metrics} FROM ${factTableRef()} WHERE ${w.clause}`;
|
|
224
|
+
}).join(" UNION ALL ")} ORDER BY ${order} ${limit}${offset}`;
|
|
225
|
+
return {
|
|
226
|
+
table,
|
|
227
|
+
params: DEVICE_SUFFIXES.flatMap(() => w.params),
|
|
228
|
+
sql
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
const col = dimColumn(q.dimension);
|
|
232
|
+
const facet = facetPredicate(q);
|
|
233
|
+
const totalCol = q.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
|
|
234
|
+
if (q.compareRange) {
|
|
235
|
+
const wPrev = partitionWhere({
|
|
236
|
+
...q,
|
|
237
|
+
range: q.compareRange
|
|
238
|
+
}, pruned, mode);
|
|
239
|
+
const curMetrics = metricList.map(metricExpr).join(", ");
|
|
240
|
+
const prevMetrics = STD_METRICS.map((m) => metricExpr(m)).join(", ");
|
|
241
|
+
const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
|
|
242
|
+
const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
|
|
243
|
+
const variantOut = q.dimension === "queryCanonical" ? ", c.variantCount AS variantCount" : "";
|
|
244
|
+
const moverWhere = q.movers ? `WHERE ${moverClause(q.movers).where} ` : "";
|
|
245
|
+
const outerOrder = q.movers ? moverOrderByAlias(q.movers) : order;
|
|
246
|
+
const inner = `SELECT COALESCE(c.k, p.k) AS ${q.dimension}, ${curCols}, ${prevCols}${variantOut}${totalCol} FROM cur c FULL OUTER JOIN prev p ON c.k = p.k ${moverWhere}`;
|
|
247
|
+
const sql = `WITH cur AS (SELECT ${col} AS k, ${curMetrics}${variantSel} FROM ${factTableRef()} WHERE ${w.clause}${facet.sql} GROUP BY ${col}), prev AS (SELECT ${col} AS k, ${prevMetrics} FROM ${factTableRef()} WHERE ${wPrev.clause}${facet.sql} GROUP BY ${col}) SELECT * FROM (${inner}) t ORDER BY ${outerOrder} ${limit}${offset}`;
|
|
248
|
+
return {
|
|
249
|
+
table,
|
|
250
|
+
params: [
|
|
251
|
+
...w.params,
|
|
252
|
+
...facet.params,
|
|
253
|
+
...wPrev.params,
|
|
254
|
+
...facet.params
|
|
255
|
+
],
|
|
256
|
+
sql
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
const metrics = metricList.map(metricExpr).join(", ");
|
|
260
|
+
const sql = `SELECT ${dimSelect(q.dimension)}, ${metrics}${variantSel}${totalCol} FROM ${factTableRef()} WHERE ${w.clause}${facet.sql} GROUP BY ${col} ORDER BY ${order} ${limit}${offset}`;
|
|
261
|
+
return {
|
|
262
|
+
table,
|
|
263
|
+
params: [...w.params, ...facet.params],
|
|
264
|
+
sql
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function buildSingleRowLookup(q, pruned, mode) {
|
|
268
|
+
const dims = Object.keys(q.match);
|
|
269
|
+
const table = tableForDimensions(dims);
|
|
270
|
+
const w = partitionWhere(q, pruned, mode);
|
|
271
|
+
const params = [...w.params];
|
|
272
|
+
let clause = w.clause;
|
|
273
|
+
for (const dim of dims) {
|
|
274
|
+
clause += ` AND ${dimColumn(dim)} = ?`;
|
|
275
|
+
params.push(q.match[dim]);
|
|
276
|
+
}
|
|
277
|
+
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
278
|
+
const groupBy = dims.length > 0 ? ` GROUP BY ${dims.map(dimColumn).join(", ")}` : "";
|
|
279
|
+
return {
|
|
280
|
+
table,
|
|
281
|
+
params,
|
|
282
|
+
sql: `SELECT ${dims.length > 0 ? `${dims.map(dimSelect).join(", ")}, ${metrics}` : metrics} FROM ${factTableRef()} WHERE ${clause}${groupBy}`
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
function buildMultiSeriesStackedDaily(q, pruned, mode) {
|
|
286
|
+
const table = tableForDimensions([q.seriesDimension]);
|
|
287
|
+
const w = partitionWhere(q, pruned, mode);
|
|
288
|
+
if (q.seriesDimension === "device") {
|
|
289
|
+
const selects = DEVICE_SUFFIXES.map((suffix) => {
|
|
290
|
+
const source = deviceSource(suffix);
|
|
291
|
+
return `SELECT date, '${suffix.toUpperCase()}' AS device, ${metricExprForSource(q.metric, source)} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date`;
|
|
292
|
+
});
|
|
293
|
+
return {
|
|
294
|
+
table,
|
|
295
|
+
params: DEVICE_SUFFIXES.flatMap(() => w.params),
|
|
296
|
+
sql: `${selects.join(" UNION ALL ")} ORDER BY date ASC, device ASC`
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
const col = dimColumn(q.seriesDimension);
|
|
300
|
+
return {
|
|
301
|
+
table,
|
|
302
|
+
params: w.params,
|
|
303
|
+
sql: `SELECT date, ${dimSelect(q.seriesDimension)}, ${metricExpr(q.metric)} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date, ${col} ORDER BY date ASC`
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
function buildTwoDimensionDetail(q, pruned, mode) {
|
|
307
|
+
const w = partitionWhere(q, pruned, mode);
|
|
308
|
+
const params = [...w.params];
|
|
309
|
+
let clause = w.clause;
|
|
310
|
+
if (q.filter?.page) {
|
|
311
|
+
clause += ` AND url = ?`;
|
|
312
|
+
params.push(q.filter.page);
|
|
313
|
+
}
|
|
314
|
+
if (q.filter?.query) {
|
|
315
|
+
clause += ` AND query = ?`;
|
|
316
|
+
params.push(q.filter.query);
|
|
317
|
+
}
|
|
318
|
+
const facet = facetPredicate(q);
|
|
319
|
+
clause += facet.sql;
|
|
320
|
+
params.push(...facet.params);
|
|
321
|
+
let sql = `SELECT url, query, ${(q.orderBy && !q.metrics.includes(q.orderBy.metric) ? [...q.metrics, q.orderBy.metric] : q.metrics).map(metricExpr).join(", ")} FROM ${factTableRef()} WHERE ${clause} GROUP BY url, query`;
|
|
322
|
+
if (q.orderBy) sql += ` ORDER BY ${metricAlias(q.orderBy.metric)} ${q.orderBy.dir.toUpperCase()}`;
|
|
323
|
+
if (q.limit && q.limit > 0) sql += ` LIMIT ${Math.floor(q.limit)}`;
|
|
324
|
+
return {
|
|
325
|
+
table: "page_queries",
|
|
326
|
+
params,
|
|
327
|
+
sql
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
function buildArchetypeSql(query, opts = {}) {
|
|
331
|
+
const pruned = opts.partitionPruned ?? false;
|
|
332
|
+
const mode = opts.partitionPredicateMode ?? (opts.partitionKeyEncoding === "string" ? "r2-sql-concat" : "bare");
|
|
333
|
+
switch (query.archetype) {
|
|
334
|
+
case "site-daily-timeseries": return buildSiteDailyTimeseries(query, pruned, mode);
|
|
335
|
+
case "entity-daily-timeseries": return buildEntityDailyTimeseries(query, pruned, mode);
|
|
336
|
+
case "entity-daily-sparkline": return buildEntityDailySparkline(query, pruned, mode);
|
|
337
|
+
case "top-n-breakdown": return buildTopNBreakdown(query, pruned, mode);
|
|
338
|
+
case "single-row-lookup": return buildSingleRowLookup(query, pruned, mode);
|
|
339
|
+
case "multi-series-stacked-daily": return buildMultiSeriesStackedDaily(query, pruned, mode);
|
|
340
|
+
case "two-dimension-detail": return buildTwoDimensionDetail(query, pruned, mode);
|
|
341
|
+
case "arbitrary-sql": throw new Error("buildArchetypeSql: arbitrary-sql carries caller SQL - the DuckDB executor runs it verbatim");
|
|
342
|
+
case "aux-cloud-only": throw new Error("buildArchetypeSql: aux-cloud-only is not an Iceberg query");
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
export { TABLE_PLACEHOLDER, buildArchetypeSql };
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { DuckDbIcebergExecutor } from "./duckdb-iceberg-executor.mjs";
|
|
2
|
+
import { R2SqlClient } from "./r2-sql-client.mjs";
|
|
3
|
+
import { ArchetypeQuery, ArchetypeResult, ArchetypeResultRow } from "@gscdump/contracts/archetypes";
|
|
4
|
+
import { Result } from "gscdump/result";
|
|
5
|
+
import { ServerTailDirective } from "@gscdump/contracts";
|
|
6
|
+
/** The two engines the server tail can route to. */
|
|
7
|
+
type ServerTailEngine = 'r2-sql' | 'duckdb';
|
|
8
|
+
/** Executors the dispatcher routes between. */
|
|
9
|
+
interface ServerTailDispatcherConfig {
|
|
10
|
+
r2Sql: R2SqlClient;
|
|
11
|
+
duckdb: DuckDbIcebergExecutor;
|
|
12
|
+
}
|
|
13
|
+
declare class ServerTailRoutingError extends Error {
|
|
14
|
+
name: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Errors-as-values core for {@link resolveServerTailEngine}: returns a
|
|
18
|
+
* `ServerTailRoutingError` instead of throwing when an archetype is `cloud-only`
|
|
19
|
+
* (the one caller-actionable routing failure — the consumer must route that
|
|
20
|
+
* query through the cloud endpoints, not the server tail). Pure — no I/O.
|
|
21
|
+
*/
|
|
22
|
+
declare function resolveServerTailEngineResult(query: ArchetypeQuery): Result<ServerTailEngine, ServerTailRoutingError>;
|
|
23
|
+
/**
|
|
24
|
+
* Decide which engine answers an archetype query. Pure — no I/O. Exposed so
|
|
25
|
+
* the file-resolution endpoint can compute the `ServerTailDirective.engine`
|
|
26
|
+
* with the SAME logic the dispatcher uses at execution time. Throws
|
|
27
|
+
* `ServerTailRoutingError` for a `cloud-only` archetype; see
|
|
28
|
+
* {@link resolveServerTailEngineResult} for the errors-as-values core.
|
|
29
|
+
*/
|
|
30
|
+
declare function resolveServerTailEngine(query: ArchetypeQuery): ServerTailEngine;
|
|
31
|
+
/** A configured server-tail dispatcher. */
|
|
32
|
+
interface ServerTailDispatcher {
|
|
33
|
+
/** Decide the engine for a query without running it. */
|
|
34
|
+
route: (query: ArchetypeQuery) => ServerTailEngine;
|
|
35
|
+
/**
|
|
36
|
+
* Execute a query, routing by execution class. If `directive` is supplied
|
|
37
|
+
* its `engine` is honoured only when consistent with the archetype's class
|
|
38
|
+
* (a `duckdb`-class archetype always runs on DuckDB regardless).
|
|
39
|
+
*/
|
|
40
|
+
execute: <R extends ArchetypeResultRow = ArchetypeResultRow>(query: ArchetypeQuery, directive?: ServerTailDirective) => Promise<ArchetypeResult<R>>;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Create the server-tail dispatcher. Holds an R2 SQL client and a DuckDB
|
|
44
|
+
* executor and routes every `ArchetypeQuery` to one of them.
|
|
45
|
+
*/
|
|
46
|
+
declare function createServerTailDispatcher(config: ServerTailDispatcherConfig): ServerTailDispatcher;
|
|
47
|
+
export { ServerTailDispatcher, ServerTailDispatcherConfig, ServerTailEngine, ServerTailRoutingError, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { ARCHETYPE_EXECUTION_CLASS } from "@gscdump/contracts/archetypes";
|
|
2
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
3
|
+
var ServerTailRoutingError = class extends Error {
|
|
4
|
+
name = "ServerTailRoutingError";
|
|
5
|
+
};
|
|
6
|
+
function routingErrorToException(error) {
|
|
7
|
+
return error;
|
|
8
|
+
}
|
|
9
|
+
function hasRegexFacet(query) {
|
|
10
|
+
return query.facets?.some((f) => f.op === "regex" || f.op === "notRegex") ?? false;
|
|
11
|
+
}
|
|
12
|
+
function referencesQueryDim(query) {
|
|
13
|
+
const q = query;
|
|
14
|
+
if (q.dimension === "queryCanonical" || q.seriesDimension === "queryCanonical") return true;
|
|
15
|
+
if (q.entity?.dimension === "queryCanonical") return true;
|
|
16
|
+
if (q.match && "queryCanonical" in q.match) return true;
|
|
17
|
+
return q.facets?.some((f) => f.column === "queryCanonical") ?? false;
|
|
18
|
+
}
|
|
19
|
+
function resolveServerTailEngineResult(query) {
|
|
20
|
+
const cls = ARCHETYPE_EXECUTION_CLASS[query.archetype];
|
|
21
|
+
if (cls === "cloud-only") return err(new ServerTailRoutingError(`archetype '${query.archetype}' is cloud-only — not a server-tail query`));
|
|
22
|
+
if (cls === "duckdb") return ok("duckdb");
|
|
23
|
+
if (query.archetype === "top-n-breakdown" && query.offset && query.offset > 0) return ok("duckdb");
|
|
24
|
+
if (referencesQueryDim(query)) return ok("duckdb");
|
|
25
|
+
if (hasRegexFacet(query)) return ok("duckdb");
|
|
26
|
+
return ok("r2-sql");
|
|
27
|
+
}
|
|
28
|
+
function resolveServerTailEngine(query) {
|
|
29
|
+
return unwrapResult(resolveServerTailEngineResult(query), routingErrorToException);
|
|
30
|
+
}
|
|
31
|
+
function sourceFor(engine) {
|
|
32
|
+
return engine === "r2-sql" ? "server-r2-sql" : "server-duckdb";
|
|
33
|
+
}
|
|
34
|
+
function extractTotal(rows) {
|
|
35
|
+
if (!rows.length || !("__total" in rows[0])) return { rows };
|
|
36
|
+
const totalRows = Number(rows[0].__total) || 0;
|
|
37
|
+
return {
|
|
38
|
+
rows: rows.map((r) => {
|
|
39
|
+
const { __total, ...rest } = r;
|
|
40
|
+
return rest;
|
|
41
|
+
}),
|
|
42
|
+
totalRows
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function createServerTailDispatcher(config) {
|
|
46
|
+
function route(query) {
|
|
47
|
+
return resolveServerTailEngine(query);
|
|
48
|
+
}
|
|
49
|
+
async function execute(query, directive) {
|
|
50
|
+
const engine = route(query);
|
|
51
|
+
if (directive && directive.engine !== engine && engine === "r2-sql") return runOn("duckdb", query);
|
|
52
|
+
return runOn(engine, query);
|
|
53
|
+
}
|
|
54
|
+
async function runOn(engine, query) {
|
|
55
|
+
if (engine === "r2-sql") {
|
|
56
|
+
const res = await config.r2Sql.runArchetype(query);
|
|
57
|
+
const { rows, totalRows } = extractTotal(res.rows);
|
|
58
|
+
return {
|
|
59
|
+
archetype: query.archetype,
|
|
60
|
+
rows,
|
|
61
|
+
source: sourceFor("r2-sql"),
|
|
62
|
+
meta: {
|
|
63
|
+
rowCount: rows.length,
|
|
64
|
+
queryMs: res.queryMs,
|
|
65
|
+
...totalRows !== void 0 ? { totalRows } : {}
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
const res = await config.duckdb.runArchetype(query);
|
|
70
|
+
const { rows, totalRows } = extractTotal(res.rows);
|
|
71
|
+
return {
|
|
72
|
+
archetype: query.archetype,
|
|
73
|
+
rows,
|
|
74
|
+
source: sourceFor("duckdb"),
|
|
75
|
+
meta: {
|
|
76
|
+
rowCount: rows.length,
|
|
77
|
+
queryMs: res.queryMs,
|
|
78
|
+
...totalRows !== void 0 ? { totalRows } : {}
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
route,
|
|
84
|
+
execute
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
export { ServerTailRoutingError, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult };
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { ArchetypeSqlPlan } from "./archetype-sql.mjs";
|
|
2
|
+
import { ArchetypeQuery } from "@gscdump/contracts/archetypes";
|
|
3
|
+
import { Result } from "gscdump/result";
|
|
4
|
+
/** Row returned by the DuckDB sibling. */
|
|
5
|
+
type DuckDbIcebergRow = Record<string, string | number | null>;
|
|
6
|
+
/**
|
|
7
|
+
* The minimal `DUCKDB_SVC` shape this executor needs — a structural subset of
|
|
8
|
+
* the binding in `workers-duckdb.ts` / `env.ts`. Any binding with `runSQL`
|
|
9
|
+
* satisfies it.
|
|
10
|
+
*/
|
|
11
|
+
interface DuckDbSvc {
|
|
12
|
+
runSQL: (args: {
|
|
13
|
+
sql: string;
|
|
14
|
+
deadlineAt?: number;
|
|
15
|
+
}) => Promise<{
|
|
16
|
+
rows: unknown[];
|
|
17
|
+
sql: string;
|
|
18
|
+
}>;
|
|
19
|
+
}
|
|
20
|
+
/** Configuration for the DuckDB-over-Iceberg executor. */
|
|
21
|
+
interface DuckDbIcebergExecutorConfig {
|
|
22
|
+
/** The DuckDB service binding (the sibling Worker RPC). */
|
|
23
|
+
svc: DuckDbSvc;
|
|
24
|
+
/**
|
|
25
|
+
* R2 Data Catalog warehouse identifier. The sibling resolves Iceberg table
|
|
26
|
+
* locations from `<warehouse>` + `<namespace>` + table name.
|
|
27
|
+
*/
|
|
28
|
+
warehouse: string;
|
|
29
|
+
/** Iceberg namespace the 5 fact tables live in. */
|
|
30
|
+
namespace: string;
|
|
31
|
+
/**
|
|
32
|
+
* How the sibling addresses an Iceberg table in a `FROM` clause. Defaults to
|
|
33
|
+
* DuckDB's `iceberg_scan('<warehouse>/<namespace>/<table>')`. Overridable so
|
|
34
|
+
* a sibling configured with the Iceberg REST catalog can use
|
|
35
|
+
* `iceberg_scan('<namespace>.<table>')` or an attached-catalog reference.
|
|
36
|
+
*/
|
|
37
|
+
tableRefStyle?: 'path' | 'catalog';
|
|
38
|
+
/** Per-query wall-clock deadline (ms). Default 25s. */
|
|
39
|
+
timeoutMs?: number;
|
|
40
|
+
}
|
|
41
|
+
/** Result of a DuckDB-over-Iceberg query. */
|
|
42
|
+
interface DuckDbIcebergResult {
|
|
43
|
+
rows: DuckDbIcebergRow[];
|
|
44
|
+
/** The exact SQL sent to the sibling. */
|
|
45
|
+
sql: string;
|
|
46
|
+
queryMs: number;
|
|
47
|
+
}
|
|
48
|
+
declare class DuckDbIcebergError extends Error {
|
|
49
|
+
name: string;
|
|
50
|
+
}
|
|
51
|
+
declare class DuckDbIcebergTimeoutError extends Error {
|
|
52
|
+
name: string;
|
|
53
|
+
constructor(timeoutMs: number);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The modelled, caller-actionable failure channel for a DuckDB-over-Iceberg
|
|
57
|
+
* query. As with the R2 SQL client, callers branch on which class came back: a
|
|
58
|
+
* `DuckDbIcebergTimeoutError` is the retry-able deadline overrun, a
|
|
59
|
+
* `DuckDbIcebergError` is a hard sibling-RPC failure (or the `aux-cloud-only`
|
|
60
|
+
* routing reject). The error variant IS the existing throwable class, so the
|
|
61
|
+
* throwing wrappers preserve the identity/message tests assert
|
|
62
|
+
* (`rejects.toThrow(/OOM in sibling/)`, `rejects.toThrow(DuckDbIcebergError)`).
|
|
63
|
+
*/
|
|
64
|
+
type DuckDbIcebergQueryError = DuckDbIcebergError | DuckDbIcebergTimeoutError;
|
|
65
|
+
/** A configured DuckDB-over-Iceberg executor. */
|
|
66
|
+
interface DuckDbIcebergExecutor {
|
|
67
|
+
/** Run a raw SQL string with `{{TABLE_<name>}}` placeholders resolved. */
|
|
68
|
+
runSql: (sql: string, params?: readonly unknown[]) => Promise<DuckDbIcebergResult>;
|
|
69
|
+
/** Run a dialect-neutral plan: resolve `{{TABLE}}`, bind params, send. */
|
|
70
|
+
runPlan: (plan: ArchetypeSqlPlan) => Promise<DuckDbIcebergResult>;
|
|
71
|
+
/** Translate + run an archetype query. Handles `arbitrary-sql` verbatim. */
|
|
72
|
+
runArchetype: (query: ArchetypeQuery) => Promise<DuckDbIcebergResult>;
|
|
73
|
+
/**
|
|
74
|
+
* Errors-as-values core for {@link DuckDbIcebergExecutor.runArchetype}:
|
|
75
|
+
* returns the modelled timeout-vs-hard-fail `DuckDbIcebergQueryError` instead
|
|
76
|
+
* of throwing, so the dispatcher can branch on retry-ability (a timeout may be
|
|
77
|
+
* worth a fallback) without `instanceof` over a `catch`. Optional so a
|
|
78
|
+
* hand-rolled executor (e.g. a host app's own service-binding executor) can
|
|
79
|
+
* implement only the throwing surface; {@link createDuckDbIcebergExecutor}
|
|
80
|
+
* always provides it.
|
|
81
|
+
*/
|
|
82
|
+
runArchetypeResult?: (query: ArchetypeQuery) => Promise<Result<DuckDbIcebergResult, DuckDbIcebergQueryError>>;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Create a DuckDB-over-Iceberg-files executor.
|
|
86
|
+
*/
|
|
87
|
+
declare function createDuckDbIcebergExecutor(config: DuckDbIcebergExecutorConfig): DuckDbIcebergExecutor;
|
|
88
|
+
export { DuckDbIcebergError, DuckDbIcebergExecutor, DuckDbIcebergExecutorConfig, DuckDbIcebergQueryError, DuckDbIcebergResult, DuckDbIcebergRow, DuckDbIcebergTimeoutError, DuckDbSvc, createDuckDbIcebergExecutor };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { TABLE_PLACEHOLDER, buildArchetypeSql } from "./archetype-sql.mjs";
|
|
2
|
+
import { bindLiterals } from "@gscdump/engine/sql";
|
|
3
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
4
|
+
var DuckDbIcebergError = class extends Error {
|
|
5
|
+
name = "DuckDbIcebergError";
|
|
6
|
+
};
|
|
7
|
+
var DuckDbIcebergTimeoutError = class extends Error {
|
|
8
|
+
name = "DuckDbIcebergTimeoutError";
|
|
9
|
+
constructor(timeoutMs) {
|
|
10
|
+
super(`DuckDB-over-Iceberg query exceeded ${timeoutMs}ms deadline`);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function duckDbIcebergErrorToException(error) {
|
|
14
|
+
return error;
|
|
15
|
+
}
|
|
16
|
+
const DEFAULT_TIMEOUT_MS = 25e3;
|
|
17
|
+
function icebergTableRef(config, table) {
|
|
18
|
+
if (config.tableRefStyle === "catalog") return `${config.namespace}.${table}`;
|
|
19
|
+
return `iceberg_scan('${config.warehouse}/${config.namespace}/${table}')`;
|
|
20
|
+
}
|
|
21
|
+
function withDeadline(op, timeoutMs) {
|
|
22
|
+
return new Promise((resolve, reject) => {
|
|
23
|
+
const timer = setTimeout(() => reject(new DuckDbIcebergTimeoutError(timeoutMs)), timeoutMs);
|
|
24
|
+
op.then(resolve, reject).finally(() => clearTimeout(timer));
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function resolveTablePlaceholders(sql, config) {
|
|
28
|
+
return sql.replace(/\{\{(\w+)\}\}/g, (_, table) => icebergTableRef(config, table));
|
|
29
|
+
}
|
|
30
|
+
function createDuckDbIcebergExecutor(config) {
|
|
31
|
+
const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
32
|
+
async function sendResult(sql) {
|
|
33
|
+
const started = Date.now();
|
|
34
|
+
const deadlineAt = Date.now() + timeoutMs;
|
|
35
|
+
const raced = await withDeadline(config.svc.runSQL({
|
|
36
|
+
sql,
|
|
37
|
+
deadlineAt
|
|
38
|
+
}), timeoutMs).then((value) => ok(value)).catch((error) => error instanceof DuckDbIcebergTimeoutError ? err(error) : err(new DuckDbIcebergError(`DUCKDB_SVC.runSQL failed: ${error.message}`)));
|
|
39
|
+
if (!raced.ok) return raced;
|
|
40
|
+
const result = raced.value;
|
|
41
|
+
return ok({
|
|
42
|
+
rows: result.rows ?? [],
|
|
43
|
+
sql: result.sql ?? sql,
|
|
44
|
+
queryMs: Date.now() - started
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
async function send(sql) {
|
|
48
|
+
return unwrapResult(await sendResult(sql), duckDbIcebergErrorToException);
|
|
49
|
+
}
|
|
50
|
+
function runSqlResult(sql, params = []) {
|
|
51
|
+
return sendResult(bindLiterals(resolveTablePlaceholders(sql, config), params));
|
|
52
|
+
}
|
|
53
|
+
function runSql(sql, params = []) {
|
|
54
|
+
return send(bindLiterals(resolveTablePlaceholders(sql, config), params));
|
|
55
|
+
}
|
|
56
|
+
function runPlan(plan) {
|
|
57
|
+
return send(bindLiterals(plan.sql.split(TABLE_PLACEHOLDER).join(icebergTableRef(config, plan.table)), plan.params));
|
|
58
|
+
}
|
|
59
|
+
function runPlanResult(plan) {
|
|
60
|
+
return sendResult(bindLiterals(plan.sql.split(TABLE_PLACEHOLDER).join(icebergTableRef(config, plan.table)), plan.params));
|
|
61
|
+
}
|
|
62
|
+
function runArchetypeResult(query) {
|
|
63
|
+
if (query.archetype === "arbitrary-sql") return runSqlResult(query.sql, query.params ?? []);
|
|
64
|
+
if (query.archetype === "aux-cloud-only") return Promise.resolve(err(new DuckDbIcebergError("aux-cloud-only is not an Iceberg query")));
|
|
65
|
+
return runPlanResult(buildArchetypeSql(query));
|
|
66
|
+
}
|
|
67
|
+
async function runArchetype(query) {
|
|
68
|
+
return unwrapResult(await runArchetypeResult(query), duckDbIcebergErrorToException);
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
runSql,
|
|
72
|
+
runPlan,
|
|
73
|
+
runArchetype,
|
|
74
|
+
runArchetypeResult
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export { DuckDbIcebergError, DuckDbIcebergTimeoutError, createDuckDbIcebergExecutor };
|