@gscdump/cloudflare 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.mjs +50 -0
- package/dist/engine.d.mts +17 -0
- package/dist/engine.mjs +35 -0
- package/dist/env.d.mts +45 -0
- package/dist/index.d.mts +4 -89
- package/dist/index.mjs +3 -475
- package/dist/inflight-dedupe.d.mts +15 -0
- package/dist/inflight-dedupe.mjs +37 -0
- package/dist/server-tail/archetype-sql.d.mts +38 -0
- package/dist/server-tail/archetype-sql.mjs +345 -0
- package/dist/server-tail/dispatcher.d.mts +47 -0
- package/dist/server-tail/dispatcher.mjs +87 -0
- package/dist/server-tail/duckdb-iceberg-executor.d.mts +88 -0
- package/dist/server-tail/duckdb-iceberg-executor.mjs +77 -0
- package/dist/server-tail/index.d.mts +4 -249
- package/dist/server-tail/index.mjs +4 -660
- package/dist/server-tail/r2-sql-client.d.mts +89 -0
- package/dist/server-tail/r2-sql-client.mjs +159 -0
- package/dist/workers-duckdb.d.mts +19 -0
- package/dist/workers-duckdb.mjs +360 -0
- package/package.json +5 -5
|
@@ -1,661 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
const TABLE_PLACEHOLDER = "{{TABLE}}";
|
|
6
|
-
const FACT_ALIAS = "fact";
|
|
7
|
-
function factTableRef() {
|
|
8
|
-
return `${TABLE_PLACEHOLDER} AS ${FACT_ALIAS}`;
|
|
9
|
-
}
|
|
10
|
-
function dimColumn(dim) {
|
|
11
|
-
if (dim === "page") return "url";
|
|
12
|
-
if (dim === "queryCanonical") return `COALESCE((SELECT qd.query_canonical FROM query_dim qd WHERE qd.query = ${FACT_ALIAS}.query LIMIT 1), ${FACT_ALIAS}.query)`;
|
|
13
|
-
return dim;
|
|
14
|
-
}
|
|
15
|
-
function dimSelect(dim) {
|
|
16
|
-
const col = dimColumn(dim);
|
|
17
|
-
return dim === "queryCanonical" ? `${col} AS queryCanonical` : col;
|
|
18
|
-
}
|
|
19
|
-
function tableForDimensions(dims) {
|
|
20
|
-
const set = new Set(dims.filter((d) => d !== "date"));
|
|
21
|
-
if (set.has("page") && (set.has("query") || set.has("queryCanonical"))) return "page_queries";
|
|
22
|
-
if (set.has("query") || set.has("queryCanonical")) return "queries";
|
|
23
|
-
if (set.has("country")) return "countries";
|
|
24
|
-
if (set.has("device")) return "dates";
|
|
25
|
-
return "pages";
|
|
26
|
-
}
|
|
27
|
-
function tableForTopNBreakdown(q) {
|
|
28
|
-
const dims = [q.dimension];
|
|
29
|
-
for (const facet of q.facets ?? []) {
|
|
30
|
-
const hasPage = q.dimension === "page" || facet.column === "page";
|
|
31
|
-
const hasQuery = q.dimension === "query" || q.dimension === "queryCanonical" || facet.column === "query" || facet.column === "queryCanonical";
|
|
32
|
-
if (hasPage && hasQuery) dims.push(facet.column);
|
|
33
|
-
}
|
|
34
|
-
return tableForDimensions(dims);
|
|
35
|
-
}
|
|
36
|
-
function metricExpr(metric) {
|
|
37
|
-
switch (metric) {
|
|
38
|
-
case "clicks": return "SUM(clicks) AS clicks";
|
|
39
|
-
case "impressions": return "SUM(impressions) AS impressions";
|
|
40
|
-
case "ctr": return "SUM(clicks) / NULLIF(SUM(impressions), 0) AS ctr";
|
|
41
|
-
case "position": return "SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position";
|
|
42
|
-
default: throw new Error(`[archetype-sql] unknown metric: ${JSON.stringify(metric)}`);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
function metricAlias(metric) {
|
|
46
|
-
switch (metric) {
|
|
47
|
-
case "clicks":
|
|
48
|
-
case "impressions":
|
|
49
|
-
case "ctr":
|
|
50
|
-
case "position": return metric;
|
|
51
|
-
default: throw new Error(`[archetype-sql] unknown order metric: ${JSON.stringify(metric)}`);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
const DEVICE_SUFFIXES = [
|
|
55
|
-
"desktop",
|
|
56
|
-
"mobile",
|
|
57
|
-
"tablet"
|
|
58
|
-
];
|
|
59
|
-
function metricExprForSource(metric, source) {
|
|
60
|
-
switch (metric) {
|
|
61
|
-
case "clicks": return `SUM(${source.clicks}) AS clicks`;
|
|
62
|
-
case "impressions": return `SUM(${source.impressions}) AS impressions`;
|
|
63
|
-
case "ctr": return `SUM(${source.clicks}) / NULLIF(SUM(${source.impressions}), 0) AS ctr`;
|
|
64
|
-
case "position": return `SUM(${source.sumPosition}) / NULLIF(SUM(${source.impressions}), 0) + 1 AS position`;
|
|
65
|
-
default: throw new Error(`[archetype-sql] unknown metric: ${JSON.stringify(metric)}`);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
function deviceSource(suffix) {
|
|
69
|
-
return {
|
|
70
|
-
clicks: `clicks_${suffix}`,
|
|
71
|
-
impressions: `impressions_${suffix}`,
|
|
72
|
-
sumPosition: `sum_position_${suffix}`
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
const STD_METRICS = [
|
|
76
|
-
"clicks",
|
|
77
|
-
"impressions",
|
|
78
|
-
"ctr",
|
|
79
|
-
"position"
|
|
80
|
-
];
|
|
81
|
-
function coalesceMetric(metric, src, alias) {
|
|
82
|
-
const ref = `${src}.${metric}`;
|
|
83
|
-
return metric === "clicks" || metric === "impressions" ? `CAST(COALESCE(${ref}, 0) AS DOUBLE) AS ${alias}` : `COALESCE(${ref}, 0) AS ${alias}`;
|
|
84
|
-
}
|
|
85
|
-
function prevAlias(metric) {
|
|
86
|
-
return `prev${metric.charAt(0).toUpperCase()}${metric.slice(1)}`;
|
|
87
|
-
}
|
|
88
|
-
function moverClause(movers) {
|
|
89
|
-
const curClicks = "COALESCE(c.clicks, 0)";
|
|
90
|
-
const prevClicks = "COALESCE(p.clicks, 0)";
|
|
91
|
-
const curImpr = "COALESCE(c.impressions, 0)";
|
|
92
|
-
const prevImpr = "COALESCE(p.impressions, 0)";
|
|
93
|
-
switch (movers) {
|
|
94
|
-
case "improving": return { where: `${curClicks} > ${prevClicks}` };
|
|
95
|
-
case "declining": return { where: `${curClicks} < ${prevClicks}` };
|
|
96
|
-
case "new": return { where: `${prevImpr} = 0 AND ${curImpr} > 0` };
|
|
97
|
-
case "lost": return { where: `${curImpr} = 0 AND ${prevImpr} > 0` };
|
|
98
|
-
default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
function moverOrderByAlias(movers) {
|
|
102
|
-
switch (movers) {
|
|
103
|
-
case "improving": return "(clicks - prevClicks) DESC";
|
|
104
|
-
case "declining": return "(clicks - prevClicks) ASC";
|
|
105
|
-
case "new": return "clicks DESC, impressions DESC";
|
|
106
|
-
case "lost": return "prevImpressions DESC";
|
|
107
|
-
default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
function sqlStringLiteral(value) {
|
|
111
|
-
return `'${value.replace(/'/g, "''")}'`;
|
|
112
|
-
}
|
|
113
|
-
function partitionWhere(q, partitionPruned = false, mode = "bare") {
|
|
114
|
-
if (partitionPruned) return {
|
|
115
|
-
clause: "date BETWEEN ? AND ?",
|
|
116
|
-
params: [q.range.start, q.range.end]
|
|
117
|
-
};
|
|
118
|
-
return {
|
|
119
|
-
clause: `${mode === "r2-sql-concat" ? "CONCAT(site_id, '')" : "site_id"} = ? AND ${mode === "r2-sql-concat" ? "CONCAT(search_type, '')" : "search_type"} = ? AND date BETWEEN ? AND ?`,
|
|
120
|
-
params: [
|
|
121
|
-
q.siteId,
|
|
122
|
-
q.searchType,
|
|
123
|
-
q.range.start,
|
|
124
|
-
q.range.end
|
|
125
|
-
]
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
function facetPredicate(query) {
|
|
129
|
-
const facets = query.facets;
|
|
130
|
-
if (!facets?.length) return {
|
|
131
|
-
sql: "",
|
|
132
|
-
params: []
|
|
133
|
-
};
|
|
134
|
-
const parts = [];
|
|
135
|
-
const params = [];
|
|
136
|
-
for (const f of facets) {
|
|
137
|
-
const col = dimColumn(f.column);
|
|
138
|
-
switch (f.op) {
|
|
139
|
-
case "eq":
|
|
140
|
-
parts.push(`${col} = ?`);
|
|
141
|
-
params.push(f.value);
|
|
142
|
-
break;
|
|
143
|
-
case "regex":
|
|
144
|
-
parts.push(`regexp_matches(LOWER(${col}), ?)`);
|
|
145
|
-
params.push(f.value);
|
|
146
|
-
break;
|
|
147
|
-
case "notRegex":
|
|
148
|
-
parts.push(`NOT regexp_matches(LOWER(${col}), ?)`);
|
|
149
|
-
params.push(f.value);
|
|
150
|
-
break;
|
|
151
|
-
default: throw new Error(`[archetype-sql] unknown facet op: ${f.op}`);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
return {
|
|
155
|
-
sql: parts.length ? ` AND ${parts.join(" AND ")}` : "",
|
|
156
|
-
params
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
function buildSiteDailyTimeseries(q, pruned, mode) {
|
|
160
|
-
const w = partitionWhere(q, pruned, mode);
|
|
161
|
-
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
162
|
-
return {
|
|
163
|
-
table: "dates",
|
|
164
|
-
params: w.params,
|
|
165
|
-
sql: `SELECT date, ${metrics} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date ORDER BY date ASC`
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
function buildEntityDailyTimeseries(q, pruned, mode) {
|
|
169
|
-
const table = tableForDimensions([q.entity.dimension]);
|
|
170
|
-
const w = partitionWhere(q, pruned, mode);
|
|
171
|
-
const col = dimColumn(q.entity.dimension);
|
|
172
|
-
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
173
|
-
return {
|
|
174
|
-
table,
|
|
175
|
-
params: [...w.params, q.entity.value],
|
|
176
|
-
sql: `SELECT date, ${metrics} FROM ${factTableRef()} WHERE ${w.clause} AND ${col} = ? GROUP BY date ORDER BY date ASC`
|
|
177
|
-
};
|
|
178
|
-
}
|
|
179
|
-
function buildEntityDailySparkline(q, pruned, mode) {
|
|
180
|
-
const table = tableForDimensions([q.dimension]);
|
|
181
|
-
const w = partitionWhere(q, pruned, mode);
|
|
182
|
-
const col = dimColumn(q.dimension);
|
|
183
|
-
if (q.entities.length === 0) throw new Error("entity-daily-sparkline: empty entities - resolver must pre-resolve the top-N list");
|
|
184
|
-
const inList = q.entities.map(sqlStringLiteral).join(", ");
|
|
185
|
-
return {
|
|
186
|
-
table,
|
|
187
|
-
params: w.params,
|
|
188
|
-
sql: `SELECT date, ${col} AS entity, ${metricExpr(q.metric)} FROM ${factTableRef()} WHERE ${w.clause} AND ${col} IN (${inList}) GROUP BY date, ${col} ORDER BY date ASC`
|
|
189
|
-
};
|
|
190
|
-
}
|
|
191
|
-
function buildTopNBreakdown(q, pruned, mode) {
|
|
192
|
-
const table = tableForTopNBreakdown(q);
|
|
193
|
-
const w = partitionWhere(q, pruned, mode);
|
|
194
|
-
if (!q.orderBy || !q.orderBy.metric || !q.orderBy.dir) throw new Error(`[archetype-sql] top-n-breakdown requires orderBy.{metric,dir}, got: ${JSON.stringify(q.orderBy)}`);
|
|
195
|
-
const order = `${metricAlias(q.orderBy.metric)} ${q.orderBy.dir.toUpperCase()}`;
|
|
196
|
-
const limit = `LIMIT ${Math.max(0, Math.floor(q.limit))}`;
|
|
197
|
-
const offset = q.offset && q.offset > 0 ? ` OFFSET ${Math.floor(q.offset)}` : "";
|
|
198
|
-
const metricList0 = q.metrics.includes(q.orderBy.metric) ? q.metrics : [...q.metrics, q.orderBy.metric];
|
|
199
|
-
const metricList = q.compareRange && q.movers ? [.../* @__PURE__ */ new Set([
|
|
200
|
-
...metricList0,
|
|
201
|
-
"clicks",
|
|
202
|
-
"impressions"
|
|
203
|
-
])] : metricList0;
|
|
204
|
-
const variantSel = q.dimension === "queryCanonical" ? ", COUNT(DISTINCT query) AS variantCount" : "";
|
|
205
|
-
if (q.dimension === "device") {
|
|
206
|
-
if (q.compareRange) {
|
|
207
|
-
const wPrev = partitionWhere({
|
|
208
|
-
...q,
|
|
209
|
-
range: q.compareRange
|
|
210
|
-
}, pruned, mode);
|
|
211
|
-
const deviceSelects = (clause, ml) => DEVICE_SUFFIXES.map((suffix) => {
|
|
212
|
-
const source = deviceSource(suffix);
|
|
213
|
-
const metrics = ml.map((m) => metricExprForSource(m, source)).join(", ");
|
|
214
|
-
return `SELECT '${suffix.toUpperCase()}' AS device, ${metrics} FROM ${factTableRef()} WHERE ${clause}`;
|
|
215
|
-
}).join(" UNION ALL ");
|
|
216
|
-
const inner = `SELECT COALESCE(c.device, p.device) AS device, ${metricList.map((m) => coalesceMetric(m, "c", m)).join(", ")}, ${STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ")} FROM cur c FULL OUTER JOIN prev p ON c.device = p.device`;
|
|
217
|
-
const sql = `WITH cur AS (${deviceSelects(w.clause, metricList)}), prev AS (${deviceSelects(wPrev.clause, STD_METRICS)}) SELECT * FROM (${inner}) t ORDER BY ${order} ${limit}${offset}`;
|
|
218
|
-
return {
|
|
219
|
-
table,
|
|
220
|
-
params: [...DEVICE_SUFFIXES.flatMap(() => w.params), ...DEVICE_SUFFIXES.flatMap(() => wPrev.params)],
|
|
221
|
-
sql
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
|
-
const sql = `${DEVICE_SUFFIXES.map((suffix) => {
|
|
225
|
-
const source = deviceSource(suffix);
|
|
226
|
-
const metrics = metricList.map((m) => metricExprForSource(m, source)).join(", ");
|
|
227
|
-
return `SELECT '${suffix.toUpperCase()}' AS device, ${metrics} FROM ${factTableRef()} WHERE ${w.clause}`;
|
|
228
|
-
}).join(" UNION ALL ")} ORDER BY ${order} ${limit}${offset}`;
|
|
229
|
-
return {
|
|
230
|
-
table,
|
|
231
|
-
params: DEVICE_SUFFIXES.flatMap(() => w.params),
|
|
232
|
-
sql
|
|
233
|
-
};
|
|
234
|
-
}
|
|
235
|
-
const col = dimColumn(q.dimension);
|
|
236
|
-
const facet = facetPredicate(q);
|
|
237
|
-
const totalCol = q.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
|
|
238
|
-
if (q.compareRange) {
|
|
239
|
-
const wPrev = partitionWhere({
|
|
240
|
-
...q,
|
|
241
|
-
range: q.compareRange
|
|
242
|
-
}, pruned, mode);
|
|
243
|
-
const curMetrics = metricList.map(metricExpr).join(", ");
|
|
244
|
-
const prevMetrics = STD_METRICS.map((m) => metricExpr(m)).join(", ");
|
|
245
|
-
const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
|
|
246
|
-
const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
|
|
247
|
-
const variantOut = q.dimension === "queryCanonical" ? ", c.variantCount AS variantCount" : "";
|
|
248
|
-
const moverWhere = q.movers ? `WHERE ${moverClause(q.movers).where} ` : "";
|
|
249
|
-
const outerOrder = q.movers ? moverOrderByAlias(q.movers) : order;
|
|
250
|
-
const inner = `SELECT COALESCE(c.k, p.k) AS ${q.dimension}, ${curCols}, ${prevCols}${variantOut}${totalCol} FROM cur c FULL OUTER JOIN prev p ON c.k = p.k ${moverWhere}`;
|
|
251
|
-
const sql = `WITH cur AS (SELECT ${col} AS k, ${curMetrics}${variantSel} FROM ${factTableRef()} WHERE ${w.clause}${facet.sql} GROUP BY ${col}), prev AS (SELECT ${col} AS k, ${prevMetrics} FROM ${factTableRef()} WHERE ${wPrev.clause}${facet.sql} GROUP BY ${col}) SELECT * FROM (${inner}) t ORDER BY ${outerOrder} ${limit}${offset}`;
|
|
252
|
-
return {
|
|
253
|
-
table,
|
|
254
|
-
params: [
|
|
255
|
-
...w.params,
|
|
256
|
-
...facet.params,
|
|
257
|
-
...wPrev.params,
|
|
258
|
-
...facet.params
|
|
259
|
-
],
|
|
260
|
-
sql
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
const metrics = metricList.map(metricExpr).join(", ");
|
|
264
|
-
const sql = `SELECT ${dimSelect(q.dimension)}, ${metrics}${variantSel}${totalCol} FROM ${factTableRef()} WHERE ${w.clause}${facet.sql} GROUP BY ${col} ORDER BY ${order} ${limit}${offset}`;
|
|
265
|
-
return {
|
|
266
|
-
table,
|
|
267
|
-
params: [...w.params, ...facet.params],
|
|
268
|
-
sql
|
|
269
|
-
};
|
|
270
|
-
}
|
|
271
|
-
function buildSingleRowLookup(q, pruned, mode) {
|
|
272
|
-
const dims = Object.keys(q.match);
|
|
273
|
-
const table = tableForDimensions(dims);
|
|
274
|
-
const w = partitionWhere(q, pruned, mode);
|
|
275
|
-
const params = [...w.params];
|
|
276
|
-
let clause = w.clause;
|
|
277
|
-
for (const dim of dims) {
|
|
278
|
-
clause += ` AND ${dimColumn(dim)} = ?`;
|
|
279
|
-
params.push(q.match[dim]);
|
|
280
|
-
}
|
|
281
|
-
const metrics = q.metrics.map(metricExpr).join(", ");
|
|
282
|
-
const groupBy = dims.length > 0 ? ` GROUP BY ${dims.map(dimColumn).join(", ")}` : "";
|
|
283
|
-
return {
|
|
284
|
-
table,
|
|
285
|
-
params,
|
|
286
|
-
sql: `SELECT ${dims.length > 0 ? `${dims.map(dimSelect).join(", ")}, ${metrics}` : metrics} FROM ${factTableRef()} WHERE ${clause}${groupBy}`
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
function buildMultiSeriesStackedDaily(q, pruned, mode) {
|
|
290
|
-
const table = tableForDimensions([q.seriesDimension]);
|
|
291
|
-
const w = partitionWhere(q, pruned, mode);
|
|
292
|
-
if (q.seriesDimension === "device") {
|
|
293
|
-
const selects = DEVICE_SUFFIXES.map((suffix) => {
|
|
294
|
-
const source = deviceSource(suffix);
|
|
295
|
-
return `SELECT date, '${suffix.toUpperCase()}' AS device, ${metricExprForSource(q.metric, source)} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date`;
|
|
296
|
-
});
|
|
297
|
-
return {
|
|
298
|
-
table,
|
|
299
|
-
params: DEVICE_SUFFIXES.flatMap(() => w.params),
|
|
300
|
-
sql: `${selects.join(" UNION ALL ")} ORDER BY date ASC, device ASC`
|
|
301
|
-
};
|
|
302
|
-
}
|
|
303
|
-
const col = dimColumn(q.seriesDimension);
|
|
304
|
-
return {
|
|
305
|
-
table,
|
|
306
|
-
params: w.params,
|
|
307
|
-
sql: `SELECT date, ${dimSelect(q.seriesDimension)}, ${metricExpr(q.metric)} FROM ${factTableRef()} WHERE ${w.clause} GROUP BY date, ${col} ORDER BY date ASC`
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
function buildTwoDimensionDetail(q, pruned, mode) {
|
|
311
|
-
const w = partitionWhere(q, pruned, mode);
|
|
312
|
-
const params = [...w.params];
|
|
313
|
-
let clause = w.clause;
|
|
314
|
-
if (q.filter?.page) {
|
|
315
|
-
clause += ` AND url = ?`;
|
|
316
|
-
params.push(q.filter.page);
|
|
317
|
-
}
|
|
318
|
-
if (q.filter?.query) {
|
|
319
|
-
clause += ` AND query = ?`;
|
|
320
|
-
params.push(q.filter.query);
|
|
321
|
-
}
|
|
322
|
-
const facet = facetPredicate(q);
|
|
323
|
-
clause += facet.sql;
|
|
324
|
-
params.push(...facet.params);
|
|
325
|
-
let sql = `SELECT url, query, ${(q.orderBy && !q.metrics.includes(q.orderBy.metric) ? [...q.metrics, q.orderBy.metric] : q.metrics).map(metricExpr).join(", ")} FROM ${factTableRef()} WHERE ${clause} GROUP BY url, query`;
|
|
326
|
-
if (q.orderBy) sql += ` ORDER BY ${metricAlias(q.orderBy.metric)} ${q.orderBy.dir.toUpperCase()}`;
|
|
327
|
-
if (q.limit && q.limit > 0) sql += ` LIMIT ${Math.floor(q.limit)}`;
|
|
328
|
-
return {
|
|
329
|
-
table: "page_queries",
|
|
330
|
-
params,
|
|
331
|
-
sql
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
function buildArchetypeSql(query, opts = {}) {
|
|
335
|
-
const pruned = opts.partitionPruned ?? false;
|
|
336
|
-
const mode = opts.partitionPredicateMode ?? (opts.partitionKeyEncoding === "string" ? "r2-sql-concat" : "bare");
|
|
337
|
-
switch (query.archetype) {
|
|
338
|
-
case "site-daily-timeseries": return buildSiteDailyTimeseries(query, pruned, mode);
|
|
339
|
-
case "entity-daily-timeseries": return buildEntityDailyTimeseries(query, pruned, mode);
|
|
340
|
-
case "entity-daily-sparkline": return buildEntityDailySparkline(query, pruned, mode);
|
|
341
|
-
case "top-n-breakdown": return buildTopNBreakdown(query, pruned, mode);
|
|
342
|
-
case "single-row-lookup": return buildSingleRowLookup(query, pruned, mode);
|
|
343
|
-
case "multi-series-stacked-daily": return buildMultiSeriesStackedDaily(query, pruned, mode);
|
|
344
|
-
case "two-dimension-detail": return buildTwoDimensionDetail(query, pruned, mode);
|
|
345
|
-
case "arbitrary-sql": throw new Error("buildArchetypeSql: arbitrary-sql carries caller SQL - the DuckDB executor runs it verbatim");
|
|
346
|
-
case "aux-cloud-only": throw new Error("buildArchetypeSql: aux-cloud-only is not an Iceberg query");
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
var ServerTailRoutingError = class extends Error {
|
|
350
|
-
name = "ServerTailRoutingError";
|
|
351
|
-
};
|
|
352
|
-
function routingErrorToException(error) {
|
|
353
|
-
return error;
|
|
354
|
-
}
|
|
355
|
-
function hasRegexFacet(query) {
|
|
356
|
-
return query.facets?.some((f) => f.op === "regex" || f.op === "notRegex") ?? false;
|
|
357
|
-
}
|
|
358
|
-
function referencesQueryDim(query) {
|
|
359
|
-
const q = query;
|
|
360
|
-
if (q.dimension === "queryCanonical" || q.seriesDimension === "queryCanonical") return true;
|
|
361
|
-
if (q.entity?.dimension === "queryCanonical") return true;
|
|
362
|
-
if (q.match && "queryCanonical" in q.match) return true;
|
|
363
|
-
return q.facets?.some((f) => f.column === "queryCanonical") ?? false;
|
|
364
|
-
}
|
|
365
|
-
function resolveServerTailEngineResult(query) {
|
|
366
|
-
const cls = ARCHETYPE_EXECUTION_CLASS[query.archetype];
|
|
367
|
-
if (cls === "cloud-only") return err(new ServerTailRoutingError(`archetype '${query.archetype}' is cloud-only — not a server-tail query`));
|
|
368
|
-
if (cls === "duckdb") return ok("duckdb");
|
|
369
|
-
if (query.archetype === "top-n-breakdown" && query.offset && query.offset > 0) return ok("duckdb");
|
|
370
|
-
if (referencesQueryDim(query)) return ok("duckdb");
|
|
371
|
-
if (hasRegexFacet(query)) return ok("duckdb");
|
|
372
|
-
return ok("r2-sql");
|
|
373
|
-
}
|
|
374
|
-
function resolveServerTailEngine(query) {
|
|
375
|
-
return unwrapResult(resolveServerTailEngineResult(query), routingErrorToException);
|
|
376
|
-
}
|
|
377
|
-
function sourceFor(engine) {
|
|
378
|
-
return engine === "r2-sql" ? "server-r2-sql" : "server-duckdb";
|
|
379
|
-
}
|
|
380
|
-
function extractTotal(rows) {
|
|
381
|
-
if (!rows.length || !("__total" in rows[0])) return { rows };
|
|
382
|
-
const totalRows = Number(rows[0].__total) || 0;
|
|
383
|
-
return {
|
|
384
|
-
rows: rows.map((r) => {
|
|
385
|
-
const { __total, ...rest } = r;
|
|
386
|
-
return rest;
|
|
387
|
-
}),
|
|
388
|
-
totalRows
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
function createServerTailDispatcher(config) {
|
|
392
|
-
function route(query) {
|
|
393
|
-
return resolveServerTailEngine(query);
|
|
394
|
-
}
|
|
395
|
-
async function execute(query, directive) {
|
|
396
|
-
const engine = route(query);
|
|
397
|
-
if (directive && directive.engine !== engine && engine === "r2-sql") return runOn("duckdb", query);
|
|
398
|
-
return runOn(engine, query);
|
|
399
|
-
}
|
|
400
|
-
async function runOn(engine, query) {
|
|
401
|
-
if (engine === "r2-sql") {
|
|
402
|
-
const res = await config.r2Sql.runArchetype(query);
|
|
403
|
-
const { rows, totalRows } = extractTotal(res.rows);
|
|
404
|
-
return {
|
|
405
|
-
archetype: query.archetype,
|
|
406
|
-
rows,
|
|
407
|
-
source: sourceFor("r2-sql"),
|
|
408
|
-
meta: {
|
|
409
|
-
rowCount: rows.length,
|
|
410
|
-
queryMs: res.queryMs,
|
|
411
|
-
...totalRows !== void 0 ? { totalRows } : {}
|
|
412
|
-
}
|
|
413
|
-
};
|
|
414
|
-
}
|
|
415
|
-
const res = await config.duckdb.runArchetype(query);
|
|
416
|
-
const { rows, totalRows } = extractTotal(res.rows);
|
|
417
|
-
return {
|
|
418
|
-
archetype: query.archetype,
|
|
419
|
-
rows,
|
|
420
|
-
source: sourceFor("duckdb"),
|
|
421
|
-
meta: {
|
|
422
|
-
rowCount: rows.length,
|
|
423
|
-
queryMs: res.queryMs,
|
|
424
|
-
...totalRows !== void 0 ? { totalRows } : {}
|
|
425
|
-
}
|
|
426
|
-
};
|
|
427
|
-
}
|
|
428
|
-
return {
|
|
429
|
-
route,
|
|
430
|
-
execute
|
|
431
|
-
};
|
|
432
|
-
}
|
|
433
|
-
var DuckDbIcebergError = class extends Error {
|
|
434
|
-
name = "DuckDbIcebergError";
|
|
435
|
-
};
|
|
436
|
-
var DuckDbIcebergTimeoutError = class extends Error {
|
|
437
|
-
name = "DuckDbIcebergTimeoutError";
|
|
438
|
-
constructor(timeoutMs) {
|
|
439
|
-
super(`DuckDB-over-Iceberg query exceeded ${timeoutMs}ms deadline`);
|
|
440
|
-
}
|
|
441
|
-
};
|
|
442
|
-
function duckDbIcebergErrorToException(error) {
|
|
443
|
-
return error;
|
|
444
|
-
}
|
|
445
|
-
const DEFAULT_TIMEOUT_MS$1 = 25e3;
|
|
446
|
-
function icebergTableRef(config, table) {
|
|
447
|
-
if (config.tableRefStyle === "catalog") return `${config.namespace}.${table}`;
|
|
448
|
-
return `iceberg_scan('${config.warehouse}/${config.namespace}/${table}')`;
|
|
449
|
-
}
|
|
450
|
-
function withDeadline(op, timeoutMs) {
|
|
451
|
-
return new Promise((resolve, reject) => {
|
|
452
|
-
const timer = setTimeout(() => reject(new DuckDbIcebergTimeoutError(timeoutMs)), timeoutMs);
|
|
453
|
-
op.then(resolve, reject).finally(() => clearTimeout(timer));
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
function resolveTablePlaceholders(sql, config) {
|
|
457
|
-
return sql.replace(/\{\{(\w+)\}\}/g, (_, table) => icebergTableRef(config, table));
|
|
458
|
-
}
|
|
459
|
-
function createDuckDbIcebergExecutor(config) {
|
|
460
|
-
const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS$1;
|
|
461
|
-
async function sendResult(sql) {
|
|
462
|
-
const started = Date.now();
|
|
463
|
-
const deadlineAt = Date.now() + timeoutMs;
|
|
464
|
-
const raced = await withDeadline(config.svc.runSQL({
|
|
465
|
-
sql,
|
|
466
|
-
deadlineAt
|
|
467
|
-
}), timeoutMs).then((value) => ok(value)).catch((error) => error instanceof DuckDbIcebergTimeoutError ? err(error) : err(new DuckDbIcebergError(`DUCKDB_SVC.runSQL failed: ${error.message}`)));
|
|
468
|
-
if (!raced.ok) return raced;
|
|
469
|
-
const result = raced.value;
|
|
470
|
-
return ok({
|
|
471
|
-
rows: result.rows ?? [],
|
|
472
|
-
sql: result.sql ?? sql,
|
|
473
|
-
queryMs: Date.now() - started
|
|
474
|
-
});
|
|
475
|
-
}
|
|
476
|
-
async function send(sql) {
|
|
477
|
-
return unwrapResult(await sendResult(sql), duckDbIcebergErrorToException);
|
|
478
|
-
}
|
|
479
|
-
function runSqlResult(sql, params = []) {
|
|
480
|
-
return sendResult(bindLiterals(resolveTablePlaceholders(sql, config), params));
|
|
481
|
-
}
|
|
482
|
-
function runSql(sql, params = []) {
|
|
483
|
-
return send(bindLiterals(resolveTablePlaceholders(sql, config), params));
|
|
484
|
-
}
|
|
485
|
-
function runPlan(plan) {
|
|
486
|
-
return send(bindLiterals(plan.sql.split(TABLE_PLACEHOLDER).join(icebergTableRef(config, plan.table)), plan.params));
|
|
487
|
-
}
|
|
488
|
-
function runPlanResult(plan) {
|
|
489
|
-
return sendResult(bindLiterals(plan.sql.split(TABLE_PLACEHOLDER).join(icebergTableRef(config, plan.table)), plan.params));
|
|
490
|
-
}
|
|
491
|
-
function runArchetypeResult(query) {
|
|
492
|
-
if (query.archetype === "arbitrary-sql") return runSqlResult(query.sql, query.params ?? []);
|
|
493
|
-
if (query.archetype === "aux-cloud-only") return Promise.resolve(err(new DuckDbIcebergError("aux-cloud-only is not an Iceberg query")));
|
|
494
|
-
return runPlanResult(buildArchetypeSql(query));
|
|
495
|
-
}
|
|
496
|
-
async function runArchetype(query) {
|
|
497
|
-
return unwrapResult(await runArchetypeResult(query), duckDbIcebergErrorToException);
|
|
498
|
-
}
|
|
499
|
-
return {
|
|
500
|
-
runSql,
|
|
501
|
-
runPlan,
|
|
502
|
-
runArchetype,
|
|
503
|
-
runArchetypeResult
|
|
504
|
-
};
|
|
505
|
-
}
|
|
506
|
-
function r2TableRef(namespace, table) {
|
|
507
|
-
return `${namespace}.${table}`;
|
|
508
|
-
}
|
|
509
|
-
var R2SqlError = class extends Error {
|
|
510
|
-
status;
|
|
511
|
-
name = "R2SqlError";
|
|
512
|
-
constructor(message, status) {
|
|
513
|
-
super(message);
|
|
514
|
-
this.status = status;
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
var R2SqlTimeoutError = class extends Error {
|
|
518
|
-
name = "R2SqlTimeoutError";
|
|
519
|
-
constructor(timeoutMs) {
|
|
520
|
-
super(`R2 SQL query exceeded ${timeoutMs}ms deadline`);
|
|
521
|
-
}
|
|
522
|
-
};
|
|
523
|
-
function r2SqlErrorToException(error) {
|
|
524
|
-
return error;
|
|
525
|
-
}
|
|
526
|
-
const DEFAULT_API_BASE = "https://api.sql.cloudflarestorage.com/api/v1";
|
|
527
|
-
const DEFAULT_TIMEOUT_MS = 25e3;
|
|
528
|
-
const PARTITION_PREDICATE_RE = /\b(site_id|search_type)(\s*=)/g;
|
|
529
|
-
function workaroundPartitionEquality(sql) {
|
|
530
|
-
return sql.replace(PARTITION_PREDICATE_RE, (_m, col, eq) => `CONCAT(${col}, '')${eq}`);
|
|
531
|
-
}
|
|
532
|
-
function escapeSqlValue(value) {
|
|
533
|
-
if (value === null || value === void 0) return "NULL";
|
|
534
|
-
if (typeof value === "number") {
|
|
535
|
-
if (!Number.isFinite(value)) throw new R2SqlError(`cannot embed non-finite number in SQL: ${value}`);
|
|
536
|
-
return String(value);
|
|
537
|
-
}
|
|
538
|
-
if (typeof value === "bigint") return value.toString();
|
|
539
|
-
if (typeof value === "boolean") return value ? "TRUE" : "FALSE";
|
|
540
|
-
return `'${String(value).replace(/'/g, "''")}'`;
|
|
541
|
-
}
|
|
542
|
-
function inlineParams(sql, params) {
|
|
543
|
-
let out = "";
|
|
544
|
-
let paramIndex = 0;
|
|
545
|
-
let inString = false;
|
|
546
|
-
for (let i = 0; i < sql.length; i++) {
|
|
547
|
-
const ch = sql[i];
|
|
548
|
-
if (ch === "'") {
|
|
549
|
-
if (inString && sql[i + 1] === "'") {
|
|
550
|
-
out += "''";
|
|
551
|
-
i++;
|
|
552
|
-
continue;
|
|
553
|
-
}
|
|
554
|
-
inString = !inString;
|
|
555
|
-
out += ch;
|
|
556
|
-
continue;
|
|
557
|
-
}
|
|
558
|
-
if (ch === "?" && !inString) {
|
|
559
|
-
if (paramIndex >= params.length) throw new R2SqlError(`SQL has more ? placeholders than params (${params.length})`);
|
|
560
|
-
out += escapeSqlValue(params[paramIndex++]);
|
|
561
|
-
continue;
|
|
562
|
-
}
|
|
563
|
-
out += ch;
|
|
564
|
-
}
|
|
565
|
-
if (paramIndex !== params.length) throw new R2SqlError(`SQL has ${paramIndex} ? placeholders but ${params.length} params supplied`);
|
|
566
|
-
return out;
|
|
567
|
-
}
|
|
568
|
-
function normalizeRows(result) {
|
|
569
|
-
if (!result) return [];
|
|
570
|
-
if (Array.isArray(result.rows)) return result.rows;
|
|
571
|
-
if (Array.isArray(result.columns) && Array.isArray(result.data)) {
|
|
572
|
-
const cols = result.columns;
|
|
573
|
-
return result.data.map((tuple) => {
|
|
574
|
-
const row = {};
|
|
575
|
-
cols.forEach((col, idx) => {
|
|
576
|
-
row[col] = tuple[idx] ?? null;
|
|
577
|
-
});
|
|
578
|
-
return row;
|
|
579
|
-
});
|
|
580
|
-
}
|
|
581
|
-
return [];
|
|
582
|
-
}
|
|
583
|
-
function coerceIntSiteId(siteId, mapper) {
|
|
584
|
-
const mapped = mapper ? mapper(siteId) : siteId;
|
|
585
|
-
const n = Number(mapped);
|
|
586
|
-
if (!Number.isSafeInteger(n)) throw new R2SqlError("int R2 SQL catalog requires a numeric site_id partition value; pass partitionSiteId for public ids");
|
|
587
|
-
return n;
|
|
588
|
-
}
|
|
589
|
-
function withEncodedPartitions(query, encoding, siteIdMapper) {
|
|
590
|
-
if (encoding === "string") return query;
|
|
591
|
-
const scoped = query;
|
|
592
|
-
const searchType = SEARCH_TYPE_INT[scoped.searchType];
|
|
593
|
-
if (searchType === void 0) throw new R2SqlError(`unknown search_type for int R2 SQL catalog: ${String(scoped.searchType)}`);
|
|
594
|
-
return {
|
|
595
|
-
...query,
|
|
596
|
-
siteId: coerceIntSiteId(scoped.siteId, siteIdMapper),
|
|
597
|
-
searchType
|
|
598
|
-
};
|
|
599
|
-
}
|
|
600
|
-
function createR2SqlClient(config) {
|
|
601
|
-
const fetchImpl = config.fetchImpl ?? globalThis.fetch;
|
|
602
|
-
const apiBase = config.apiBase ?? DEFAULT_API_BASE;
|
|
603
|
-
const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
604
|
-
const partitionKeyEncoding = config.partitionKeyEncoding ?? "int";
|
|
605
|
-
const endpoint = `${apiBase}/accounts/${config.accountId}/r2-sql/query/${config.bucket}`;
|
|
606
|
-
async function queryResult(sql) {
|
|
607
|
-
const started = Date.now();
|
|
608
|
-
const controller = new AbortController();
|
|
609
|
-
const timer = setTimeout(() => controller.abort(new R2SqlTimeoutError(timeoutMs)), timeoutMs);
|
|
610
|
-
let response;
|
|
611
|
-
try {
|
|
612
|
-
response = await fetchImpl(endpoint, {
|
|
613
|
-
method: "POST",
|
|
614
|
-
headers: {
|
|
615
|
-
"authorization": `Bearer ${config.token}`,
|
|
616
|
-
"content-type": "application/json",
|
|
617
|
-
"user-agent": "gscdump-cloudflare-r2sql/1.0"
|
|
618
|
-
},
|
|
619
|
-
body: JSON.stringify({ query: sql }),
|
|
620
|
-
signal: controller.signal
|
|
621
|
-
});
|
|
622
|
-
} catch (error) {
|
|
623
|
-
if (error instanceof R2SqlTimeoutError || error?.name === "AbortError") return err(new R2SqlTimeoutError(timeoutMs));
|
|
624
|
-
return err(new R2SqlError(`R2 SQL request failed: ${error.message}`));
|
|
625
|
-
} finally {
|
|
626
|
-
clearTimeout(timer);
|
|
627
|
-
}
|
|
628
|
-
if (!response.ok) {
|
|
629
|
-
const text = await response.text().catch(() => "");
|
|
630
|
-
return err(new R2SqlError(`R2 SQL HTTP ${response.status}: ${text}`, response.status));
|
|
631
|
-
}
|
|
632
|
-
const envelope = await response.json();
|
|
633
|
-
if (!envelope.success) return err(new R2SqlError(`R2 SQL query rejected: ${envelope.errors?.map((e) => e.message).join("; ") ?? "unknown R2 SQL error"}`));
|
|
634
|
-
return ok({
|
|
635
|
-
rows: normalizeRows(envelope.result),
|
|
636
|
-
sql,
|
|
637
|
-
queryMs: Date.now() - started
|
|
638
|
-
});
|
|
639
|
-
}
|
|
640
|
-
async function query(sql) {
|
|
641
|
-
return unwrapResult(await queryResult(sql), r2SqlErrorToException);
|
|
642
|
-
}
|
|
643
|
-
function materializePlan(plan) {
|
|
644
|
-
const tableRef = r2TableRef(config.namespace, plan.table);
|
|
645
|
-
return inlineParams(plan.sql.split(TABLE_PLACEHOLDER).join(tableRef), plan.params);
|
|
646
|
-
}
|
|
647
|
-
function runPlan(plan) {
|
|
648
|
-
const sql = materializePlan(plan);
|
|
649
|
-
return query(partitionKeyEncoding === "string" ? workaroundPartitionEquality(sql) : sql);
|
|
650
|
-
}
|
|
651
|
-
function runArchetype(archetypeQuery) {
|
|
652
|
-
return query(materializePlan(buildArchetypeSql(withEncodedPartitions(archetypeQuery, partitionKeyEncoding, config.partitionSiteId), { partitionKeyEncoding })));
|
|
653
|
-
}
|
|
654
|
-
return {
|
|
655
|
-
query,
|
|
656
|
-
queryResult,
|
|
657
|
-
runPlan,
|
|
658
|
-
runArchetype
|
|
659
|
-
};
|
|
660
|
-
}
|
|
1
|
+
import { TABLE_PLACEHOLDER, buildArchetypeSql } from "./archetype-sql.mjs";
|
|
2
|
+
import { ServerTailRoutingError, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult } from "./dispatcher.mjs";
|
|
3
|
+
import { DuckDbIcebergError, DuckDbIcebergTimeoutError, createDuckDbIcebergExecutor } from "./duckdb-iceberg-executor.mjs";
|
|
4
|
+
import { R2SqlError, R2SqlTimeoutError, createR2SqlClient } from "./r2-sql-client.mjs";
|
|
661
5
|
export { DuckDbIcebergError, DuckDbIcebergTimeoutError, R2SqlError, R2SqlTimeoutError, ServerTailRoutingError, TABLE_PLACEHOLDER, buildArchetypeSql, createDuckDbIcebergExecutor, createR2SqlClient, createServerTailDispatcher, resolveServerTailEngine, resolveServerTailEngineResult };
|