@gscdump/engine 0.11.5 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/coerce.mjs +14 -0
- package/dist/_chunks/dispatch.mjs +1 -0
- package/dist/_chunks/duckdb.d.mts +1 -1
- package/dist/_chunks/engine.mjs +1 -129
- package/dist/_chunks/planner.d.mts +1 -1
- package/dist/_chunks/registry.d.mts +8 -9
- package/dist/_chunks/resolver.mjs +704 -3
- package/dist/_chunks/storage.d.mts +1 -50
- package/dist/_chunks/types.d.mts +1 -0
- package/dist/adapters/filesystem.d.mts +1 -1
- package/dist/adapters/hyparquet.d.mts +1 -1
- package/dist/adapters/node.d.mts +1 -1
- package/dist/adapters/r2-manifest.d.mts +1 -1
- package/dist/adapters/r2.d.mts +1 -1
- package/dist/contracts.d.mts +1 -1
- package/dist/entities.d.mts +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +1 -13
- package/dist/ingest.d.mts +1 -1
- package/dist/planner.d.mts +1 -1
- package/dist/resolver/index.d.mts +1 -0
- package/dist/resolver/index.mjs +1 -2
- package/dist/rollups.d.mts +1 -1
- package/dist/source/index.d.mts +5 -4
- package/dist/source/index.mjs +9 -12
- package/package.json +3 -3
- package/dist/_chunks/pg-adapter.mjs +0 -667
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
function coerceRow(row) {
|
|
2
|
+
let mutated = null;
|
|
3
|
+
for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
|
|
4
|
+
if (!mutated) mutated = { ...row };
|
|
5
|
+
mutated[k] = Number(v);
|
|
6
|
+
}
|
|
7
|
+
return mutated ?? row;
|
|
8
|
+
}
|
|
9
|
+
function coerceRows(rows) {
|
|
10
|
+
const out = Array.from({ length: rows.length });
|
|
11
|
+
for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
export { coerceRows as n, coerceRow as t };
|
|
@@ -9,6 +9,7 @@ var AnalyzerCapabilityError = class extends Error {
|
|
|
9
9
|
}
|
|
10
10
|
};
|
|
11
11
|
function sourceHas(source, cap) {
|
|
12
|
+
if (cap === "executeSql") return typeof source.executeSql === "function";
|
|
12
13
|
return source.capabilities[cap] === true;
|
|
13
14
|
}
|
|
14
15
|
function assertSatisfies(analyzer, source) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { S as Row, b as QueryExecutor, j as TableName, p as ParquetCodec } from "./storage.mjs";
|
|
2
2
|
interface DuckDBHandle {
|
|
3
3
|
query: (sql: string, params?: unknown[]) => Promise<Row[]>;
|
|
4
4
|
registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
|
package/dist/_chunks/engine.mjs
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { r as currentSchemaVersion, t as SCHEMAS } from "./schema.mjs";
|
|
2
2
|
import { i as inferSearchType, n as dayPartition, s as objectKey, u as tenantPrefix } from "./storage.mjs";
|
|
3
|
-
import { a as compactTieredImpl, i as substituteNamedFiles, n as compileLogicalQueryPlan
|
|
3
|
+
import { a as compactTieredImpl, i as substituteNamedFiles, n as compileLogicalQueryPlan } from "./compiler.mjs";
|
|
4
4
|
import { sqlEscape } from "../sql-bind.mjs";
|
|
5
|
-
import { d as buildExtrasQueries, f as buildTotalsSql, g as resolveToSQLOptimized, m as resolveComparisonSQL, t as createParquetResolverAdapter } from "./pg-adapter.mjs";
|
|
6
5
|
import { buildLogicalPlan } from "gscdump/query/plan";
|
|
7
6
|
import { normalizeUrl } from "gscdump/normalize";
|
|
8
7
|
async function encodeBytes(db, table, rows) {
|
|
@@ -372,130 +371,6 @@ function createStorageEngine(opts) {
|
|
|
372
371
|
signal: ctx.signal
|
|
373
372
|
});
|
|
374
373
|
}
|
|
375
|
-
async function queryComparison(ctx, current, previous, filter) {
|
|
376
|
-
const adapter = createParquetResolverAdapter();
|
|
377
|
-
const currentPlan = buildLogicalPlan(current, adapter.capabilities);
|
|
378
|
-
const previousPlan = buildLogicalPlan(previous, adapter.capabilities);
|
|
379
|
-
if (currentPlan.dataset !== previousPlan.dataset) throw new Error(`queryComparison: current (${currentPlan.dataset}) and previous (${previousPlan.dataset}) must resolve to the same table`);
|
|
380
|
-
const table = ctx.table ?? currentPlan.dataset;
|
|
381
|
-
const comparison = resolveComparisonSQL(current, previous, {
|
|
382
|
-
adapter,
|
|
383
|
-
siteId: void 0
|
|
384
|
-
}, filter);
|
|
385
|
-
const totals = buildTotalsSql(current, {
|
|
386
|
-
adapter,
|
|
387
|
-
siteId: void 0
|
|
388
|
-
});
|
|
389
|
-
const fileSets = { FILES: {
|
|
390
|
-
table,
|
|
391
|
-
partitions: enumeratePartitions(currentPlan.dateRange.startDate < previousPlan.dateRange.startDate ? currentPlan.dateRange.startDate : previousPlan.dateRange.startDate, currentPlan.dateRange.endDate > previousPlan.dateRange.endDate ? currentPlan.dateRange.endDate : previousPlan.dateRange.endDate)
|
|
392
|
-
} };
|
|
393
|
-
const baseCtx = {
|
|
394
|
-
userId: ctx.userId,
|
|
395
|
-
siteId: ctx.siteId
|
|
396
|
-
};
|
|
397
|
-
const [main, count, totalsRow] = await Promise.all([
|
|
398
|
-
runSQL({
|
|
399
|
-
ctx: baseCtx,
|
|
400
|
-
table,
|
|
401
|
-
fileSets,
|
|
402
|
-
sql: comparison.sql,
|
|
403
|
-
params: comparison.params,
|
|
404
|
-
signal: ctx.signal
|
|
405
|
-
}),
|
|
406
|
-
runSQL({
|
|
407
|
-
ctx: baseCtx,
|
|
408
|
-
table,
|
|
409
|
-
fileSets,
|
|
410
|
-
sql: comparison.countSql,
|
|
411
|
-
params: comparison.countParams,
|
|
412
|
-
signal: ctx.signal
|
|
413
|
-
}),
|
|
414
|
-
runSQL({
|
|
415
|
-
ctx: baseCtx,
|
|
416
|
-
table,
|
|
417
|
-
fileSets,
|
|
418
|
-
sql: totals.sql,
|
|
419
|
-
params: totals.params,
|
|
420
|
-
signal: ctx.signal
|
|
421
|
-
})
|
|
422
|
-
]);
|
|
423
|
-
return {
|
|
424
|
-
rows: main.rows,
|
|
425
|
-
totalCount: Number(count.rows[0]?.total ?? 0),
|
|
426
|
-
totals: totalsRow.rows[0] ?? {}
|
|
427
|
-
};
|
|
428
|
-
}
|
|
429
|
-
async function queryOptimized(ctx, state) {
|
|
430
|
-
const adapter = createParquetResolverAdapter();
|
|
431
|
-
const plan = buildLogicalPlan(state, adapter.capabilities);
|
|
432
|
-
const table = ctx.table ?? plan.dataset;
|
|
433
|
-
const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
|
|
434
|
-
const { sql, params } = resolveToSQLOptimized(state, {
|
|
435
|
-
adapter,
|
|
436
|
-
siteId: void 0
|
|
437
|
-
});
|
|
438
|
-
const result = await runSQL({
|
|
439
|
-
ctx: {
|
|
440
|
-
userId: ctx.userId,
|
|
441
|
-
siteId: ctx.siteId
|
|
442
|
-
},
|
|
443
|
-
table,
|
|
444
|
-
fileSets: { FILES: {
|
|
445
|
-
table,
|
|
446
|
-
partitions
|
|
447
|
-
} },
|
|
448
|
-
sql,
|
|
449
|
-
params,
|
|
450
|
-
signal: ctx.signal
|
|
451
|
-
});
|
|
452
|
-
const firstRow = result.rows[0];
|
|
453
|
-
const totalCount = Number(firstRow?.totalCount ?? 0);
|
|
454
|
-
const totals = {
|
|
455
|
-
clicks: Number(firstRow?.totalClicks ?? 0),
|
|
456
|
-
impressions: Number(firstRow?.totalImpressions ?? 0),
|
|
457
|
-
ctr: Number(firstRow?.totalCtr ?? 0),
|
|
458
|
-
position: Number(firstRow?.totalPosition ?? 0)
|
|
459
|
-
};
|
|
460
|
-
return {
|
|
461
|
-
rows: result.rows.map((r) => {
|
|
462
|
-
const { totalCount: _tc, totalClicks: _tcl, totalImpressions: _ti, totalCtr: _tr, totalPosition: _tp, ...rest } = r;
|
|
463
|
-
return rest;
|
|
464
|
-
}),
|
|
465
|
-
totalCount,
|
|
466
|
-
totals
|
|
467
|
-
};
|
|
468
|
-
}
|
|
469
|
-
async function queryExtras(ctx, state) {
|
|
470
|
-
const adapter = createParquetResolverAdapter();
|
|
471
|
-
const extras = buildExtrasQueries(state, {
|
|
472
|
-
adapter,
|
|
473
|
-
siteId: void 0
|
|
474
|
-
});
|
|
475
|
-
if (extras.length === 0) return [];
|
|
476
|
-
const plan = buildLogicalPlan(state, adapter.capabilities);
|
|
477
|
-
const table = ctx.table ?? plan.dataset;
|
|
478
|
-
const fileSets = { FILES: {
|
|
479
|
-
table,
|
|
480
|
-
partitions: enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate)
|
|
481
|
-
} };
|
|
482
|
-
const baseCtx = {
|
|
483
|
-
userId: ctx.userId,
|
|
484
|
-
siteId: ctx.siteId
|
|
485
|
-
};
|
|
486
|
-
const results = await Promise.all(extras.map((e) => runSQL({
|
|
487
|
-
ctx: baseCtx,
|
|
488
|
-
table,
|
|
489
|
-
fileSets,
|
|
490
|
-
sql: e.sql,
|
|
491
|
-
params: e.params,
|
|
492
|
-
signal: ctx.signal
|
|
493
|
-
})));
|
|
494
|
-
return extras.map((e, i) => ({
|
|
495
|
-
key: e.key,
|
|
496
|
-
rows: results[i].rows
|
|
497
|
-
}));
|
|
498
|
-
}
|
|
499
374
|
async function compactTiered(ctx, thresholds) {
|
|
500
375
|
return compactTieredImpl({
|
|
501
376
|
dataSource,
|
|
@@ -604,9 +479,6 @@ function createStorageEngine(opts) {
|
|
|
604
479
|
return {
|
|
605
480
|
writeDay,
|
|
606
481
|
query,
|
|
607
|
-
queryComparison,
|
|
608
|
-
queryExtras,
|
|
609
|
-
queryOptimized,
|
|
610
482
|
runSQL,
|
|
611
483
|
compactTiered,
|
|
612
484
|
gcOrphans,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { o as ResolverAdapter } from "./types.mjs";
|
|
1
|
+
import { S as Row$1 } from "./storage.mjs";
|
|
3
2
|
import { t as AnalysisParams } from "./analysis-types.mjs";
|
|
3
|
+
import { o as ResolverAdapter } from "./types.mjs";
|
|
4
4
|
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
5
5
|
import { TableName } from "@gscdump/contracts";
|
|
6
6
|
import { BuilderState } from "gscdump/query";
|
|
@@ -14,12 +14,10 @@ interface ExecuteSqlOptions {
|
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* Flat capability bag: planner-side flags (`regex`, `comparisonJoin`, ...)
|
|
17
|
-
* mixed with storage-side flags.
|
|
18
|
-
*
|
|
19
|
-
* of probing the function shape.
|
|
17
|
+
* mixed with storage-side flags. SQL execution is not a capability flag —
|
|
18
|
+
* callers probe `typeof source.executeSql === 'function'`.
|
|
20
19
|
*/
|
|
21
20
|
interface SourceCapabilities extends PlannerCapabilities {
|
|
22
|
-
executeSql?: boolean;
|
|
23
21
|
attachedTables?: boolean;
|
|
24
22
|
fileSets?: boolean;
|
|
25
23
|
/**
|
|
@@ -43,9 +41,10 @@ interface AnalysisQuerySource {
|
|
|
43
41
|
siteId?: string | number;
|
|
44
42
|
queryRows: (state: BuilderState) => Promise<QueryRow[]>;
|
|
45
43
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
44
|
+
* Optional raw-SQL escape hatch. Receives the compiled SQL plan with
|
|
45
|
+
* `{{FILES}}` placeholders; sources that advertise `capabilities.fileSets`
|
|
46
|
+
* consume `opts.fileSets`, others ignore them. Implementations MUST coerce
|
|
47
|
+
* BigInts to numbers before returning (see Source invariant above).
|
|
49
48
|
*/
|
|
50
49
|
executeSql?: (sql: string, params?: unknown[], opts?: ExecuteSqlOptions) => Promise<QueryRow[]>;
|
|
51
50
|
}
|