@gscdump/engine 0.21.3 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/analysis-types.d.mts +1 -1
- package/dist/_chunks/coerce.mjs +1 -1
- package/dist/_chunks/dispatch.mjs +1 -1
- package/dist/_chunks/duckdb.d.mts +2 -2
- package/dist/_chunks/engine.mjs +4 -4
- package/dist/_chunks/iceberg-schema.mjs +7 -3
- package/dist/_chunks/index.d.mts +2 -2
- package/dist/_chunks/parquet-plan.mjs +3 -3
- package/dist/_chunks/planner.d.mts +2 -2
- package/dist/_chunks/registry.d.mts +4 -4
- package/dist/_chunks/resolver.mjs +60 -3
- package/dist/_chunks/schema.d.mts +1067 -275
- package/dist/_chunks/schema.mjs +70 -2
- package/dist/_chunks/sink.d.mts +49 -10
- package/dist/_chunks/snapshot.d.mts +1 -1
- package/dist/_chunks/storage.d.mts +1 -1
- package/dist/_chunks/storage.mjs +1 -1
- package/dist/_chunks/types.d.mts +1 -1
- package/dist/adapters/duckdb-node.d.mts +1 -1
- package/dist/adapters/filesystem.d.mts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/hyparquet.d.mts +2 -2
- package/dist/adapters/hyparquet.mjs +1 -1
- package/dist/adapters/node.d.mts +2 -2
- package/dist/adapters/node.mjs +1 -1
- package/dist/adapters/r2-manifest.d.mts +1 -1
- package/dist/adapters/r2-manifest.mjs +1 -1
- package/dist/adapters/r2.d.mts +1 -1
- package/dist/analysis-types.d.mts +1 -1
- package/dist/analyzer/index.d.mts +2 -2
- package/dist/analyzer/index.mjs +1 -1
- package/dist/contracts.d.mts +1 -1
- package/dist/entities.d.mts +1 -1
- package/dist/index.d.mts +6 -6
- package/dist/index.mjs +64 -8
- package/dist/ingest.d.mts +5 -1
- package/dist/ingest.mjs +57 -1
- package/dist/period/index.d.mts +1 -1
- package/dist/planner.d.mts +2 -2
- package/dist/planner.mjs +1 -1
- package/dist/report/index.d.mts +2 -2
- package/dist/resolver/index.d.mts +2 -2
- package/dist/resolver/index.mjs +1 -1
- package/dist/rollups.d.mts +2 -2
- package/dist/schema.d.mts +2 -2
- package/dist/schema.mjs +2 -2
- package/dist/sink-node.d.mts +1 -1
- package/dist/sink-node.mjs +1 -1
- package/dist/snapshot.d.mts +1 -1
- package/dist/source/index.d.mts +4 -4
- package/dist/source/index.mjs +3 -3
- package/package.json +6 -6
|
@@ -46,4 +46,4 @@ interface AnalysisResult {
|
|
|
46
46
|
}
|
|
47
47
|
/** Coerce arbitrary value (number, bigint, string, null) to number, defaulting to 0. */
|
|
48
48
|
declare function num(v: unknown): number;
|
|
49
|
-
export {
|
|
49
|
+
export { AnalysisParams, AnalysisResult, AnalysisTool, num };
|
package/dist/_chunks/coerce.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ParquetCodec, QueryExecutor, Row, TableName } from "./storage.mjs";
|
|
2
2
|
interface DuckDBHandle {
|
|
3
3
|
query: (sql: string, params?: unknown[]) => Promise<Row[]>;
|
|
4
4
|
registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
|
|
@@ -23,4 +23,4 @@ declare function createDuckDBExecutor(factory: DuckDBFactory): QueryExecutor;
|
|
|
23
23
|
* to satisfy the ParquetCodec empty-rows invariant.
|
|
24
24
|
*/
|
|
25
25
|
declare function canonicalEmptyParquetSchema(table: TableName): string;
|
|
26
|
-
export {
|
|
26
|
+
export { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor };
|
package/dist/_chunks/engine.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
|
|
2
|
+
import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./storage.mjs";
|
|
3
|
+
import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
|
|
4
4
|
import { sqlEscape } from "../sql-bind.mjs";
|
|
5
5
|
import { buildLogicalPlan } from "gscdump/query/plan";
|
|
6
6
|
import { normalizeUrl } from "gscdump/normalize";
|
|
@@ -601,4 +601,4 @@ function createStorageEngine(opts) {
|
|
|
601
601
|
readObject: (key) => dataSource.read(key)
|
|
602
602
|
};
|
|
603
603
|
}
|
|
604
|
-
export {
|
|
604
|
+
export { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine, gcOrphansImpl };
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { SCHEMAS } from "./schema.mjs";
|
|
2
2
|
const ICEBERG_TABLES = [
|
|
3
3
|
"pages",
|
|
4
4
|
"queries",
|
|
5
5
|
"countries",
|
|
6
6
|
"page_queries",
|
|
7
|
-
"dates"
|
|
7
|
+
"dates",
|
|
8
|
+
"search_appearance",
|
|
9
|
+
"search_appearance_pages",
|
|
10
|
+
"search_appearance_queries",
|
|
11
|
+
"search_appearance_page_queries"
|
|
8
12
|
];
|
|
9
13
|
const ICEBERG_PARTITION_COLUMNS = [{
|
|
10
14
|
name: "site_id",
|
|
@@ -64,4 +68,4 @@ function icebergTableSpec(table) {
|
|
|
64
68
|
};
|
|
65
69
|
}
|
|
66
70
|
const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
|
|
67
|
-
export {
|
|
71
|
+
export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, icebergTableSpec };
|
package/dist/_chunks/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AnalysisParams } from "./analysis-types.mjs";
|
|
2
2
|
type WindowPreset = 'last-7d' | 'last-28d' | 'last-30d' | 'last-90d' | 'last-180d' | 'last-365d' | 'mtd' | 'ytd' | 'custom';
|
|
3
3
|
type ComparisonMode = 'none' | 'prev-period' | 'yoy';
|
|
4
4
|
interface ResolveWindowOptions {
|
|
@@ -54,4 +54,4 @@ type DateRowShape = Record<string, unknown> & {
|
|
|
54
54
|
* once. Existing dates keep all their rows (grouped timeseries safe).
|
|
55
55
|
*/
|
|
56
56
|
declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
|
|
57
|
-
export {
|
|
57
|
+
export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { currentSchemaVersion, dimensionToColumn } from "./schema.mjs";
|
|
2
|
+
import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./storage.mjs";
|
|
3
3
|
import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "../sql-fragments.mjs";
|
|
4
4
|
import { MS_PER_DAY } from "gscdump";
|
|
5
5
|
import { buildLogicalPlan } from "gscdump/query/plan";
|
|
@@ -381,4 +381,4 @@ function substituteNamedFiles(sql, sets) {
|
|
|
381
381
|
for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
|
|
382
382
|
return out;
|
|
383
383
|
}
|
|
384
|
-
export {
|
|
384
|
+
export { FILES_PLACEHOLDER, RAW_DAILY_COMPACT_THRESHOLD, compactTieredImpl, compileLogicalQueryPlan, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, splitOverlappingTiers, substituteNamedFiles };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TableName } from "./storage.mjs";
|
|
2
2
|
import { LogicalQueryPlan } from "gscdump/query/plan";
|
|
3
3
|
import { BuilderState } from "gscdump/query";
|
|
4
4
|
interface ResolvedQuery {
|
|
@@ -12,4 +12,4 @@ declare const FILES_PLACEHOLDER = "{{FILES}}";
|
|
|
12
12
|
declare function compileLogicalQueryPlan(plan: LogicalQueryPlan, table?: TableName): ResolvedQuery;
|
|
13
13
|
declare function resolveParquetSQL(state: BuilderState, table?: TableName): ResolvedQuery;
|
|
14
14
|
declare function substituteNamedFiles(sql: string, sets: Record<string, string[]>): string;
|
|
15
|
-
export {
|
|
15
|
+
export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { Row as Row$1 } from "./storage.mjs";
|
|
2
|
+
import { AnalysisParams } from "./analysis-types.mjs";
|
|
3
|
+
import { ResolverAdapter } from "./types.mjs";
|
|
4
4
|
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
5
5
|
import { TableName } from "@gscdump/contracts";
|
|
6
6
|
import { BuilderState } from "gscdump/query";
|
|
@@ -210,4 +210,4 @@ interface AnalyzerRegistry {
|
|
|
210
210
|
* or per-request in a worker).
|
|
211
211
|
*/
|
|
212
212
|
declare function createAnalyzerRegistry(init?: AnalyzerRegistryInit): AnalyzerRegistry;
|
|
213
|
-
export {
|
|
213
|
+
export { AnalysisQuerySource, AnalysisSourceKind, Analyzer, AnalyzerRegistry, AnalyzerRegistryInit, AnalyzerVariants, BuildContext, DefineAnalyzerOptions, DefinedAnalyzer, ExecuteSqlOptions, FileSet, Plan, QueryRow, ReduceContext, ReduceCtx, Reducer, RequiredCapability, RowQueriesPlan, SourceCapabilities, SqlExtraQuery, SqlPlan, SqlPlanSpec, TypedRowQuery, createAnalyzerRegistry, defineAnalyzer, requireAdapter };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { SCHEMAS, drizzleSchema } from "./schema.mjs";
|
|
2
|
+
import { enumeratePartitions } from "./parquet-plan.mjs";
|
|
3
3
|
import { escapeLike } from "../sql-fragments.mjs";
|
|
4
4
|
import "../planner.mjs";
|
|
5
5
|
import { PgDialect } from "drizzle-orm/pg-core";
|
|
@@ -83,6 +83,60 @@ const LOGICAL_DATASETS = {
|
|
|
83
83
|
surfaces: ["api", "stored"]
|
|
84
84
|
}
|
|
85
85
|
} },
|
|
86
|
+
search_appearance_pages: { dimensions: {
|
|
87
|
+
searchAppearance: {
|
|
88
|
+
column: "searchAppearance",
|
|
89
|
+
surfaces: ["stored"]
|
|
90
|
+
},
|
|
91
|
+
page: {
|
|
92
|
+
column: "url",
|
|
93
|
+
surfaces: ["stored"]
|
|
94
|
+
},
|
|
95
|
+
date: {
|
|
96
|
+
column: "date",
|
|
97
|
+
surfaces: ["stored"]
|
|
98
|
+
}
|
|
99
|
+
} },
|
|
100
|
+
search_appearance_queries: { dimensions: {
|
|
101
|
+
searchAppearance: {
|
|
102
|
+
column: "searchAppearance",
|
|
103
|
+
surfaces: ["stored"]
|
|
104
|
+
},
|
|
105
|
+
query: {
|
|
106
|
+
column: "query",
|
|
107
|
+
surfaces: ["stored"]
|
|
108
|
+
},
|
|
109
|
+
queryCanonical: {
|
|
110
|
+
column: "query_canonical",
|
|
111
|
+
surfaces: ["stored", "derived"]
|
|
112
|
+
},
|
|
113
|
+
date: {
|
|
114
|
+
column: "date",
|
|
115
|
+
surfaces: ["stored"]
|
|
116
|
+
}
|
|
117
|
+
} },
|
|
118
|
+
search_appearance_page_queries: { dimensions: {
|
|
119
|
+
searchAppearance: {
|
|
120
|
+
column: "searchAppearance",
|
|
121
|
+
surfaces: ["stored"]
|
|
122
|
+
},
|
|
123
|
+
page: {
|
|
124
|
+
column: "url",
|
|
125
|
+
surfaces: ["stored"]
|
|
126
|
+
},
|
|
127
|
+
query: {
|
|
128
|
+
column: "query",
|
|
129
|
+
surfaces: ["stored"]
|
|
130
|
+
},
|
|
131
|
+
queryCanonical: {
|
|
132
|
+
column: "query_canonical",
|
|
133
|
+
surfaces: ["stored", "derived"]
|
|
134
|
+
},
|
|
135
|
+
date: {
|
|
136
|
+
column: "date",
|
|
137
|
+
surfaces: ["stored"]
|
|
138
|
+
}
|
|
139
|
+
} },
|
|
86
140
|
hourly_pages: { dimensions: {
|
|
87
141
|
page: {
|
|
88
142
|
column: "url",
|
|
@@ -734,6 +788,9 @@ const PG_BASE_CONFIG = {
|
|
|
734
788
|
countries: "countries",
|
|
735
789
|
dates: "dates",
|
|
736
790
|
search_appearance: "search_appearance",
|
|
791
|
+
search_appearance_pages: "search_appearance_pages",
|
|
792
|
+
search_appearance_queries: "search_appearance_queries",
|
|
793
|
+
search_appearance_page_queries: "search_appearance_page_queries",
|
|
737
794
|
hourly_pages: "hourly_pages"
|
|
738
795
|
},
|
|
739
796
|
metricCast: "DOUBLE",
|
|
@@ -855,4 +912,4 @@ function assertSchemaInSync(options) {
|
|
|
855
912
|
if (missing.length > 0 || extra.length > 0) throw new Error(`${label} drizzle schema for '${key}' drifted from SCHEMAS. Missing: [${missing.join(", ")}]. Extra: [${extra.join(", ")}].`);
|
|
856
913
|
}
|
|
857
914
|
}
|
|
858
|
-
export {
|
|
915
|
+
export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
|