@gscdump/analysis 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.mjs +51 -51
- package/dist/routing/index.d.mts +23 -0
- package/dist/routing/index.mjs +53 -0
- package/dist/source/index.d.mts +23 -3
- package/dist/source/index.mjs +102 -57
- package/package.json +10 -6
package/dist/index.mjs
CHANGED
|
@@ -2579,53 +2579,6 @@ const WHITESPACE_RE = /\s+/g;
|
|
|
2579
2579
|
function normalizeQuery(query) {
|
|
2580
2580
|
return query.toLowerCase().replace(SEPARATOR_RE, " ").replace(WHITESPACE_RE, " ").trim().split(" ").filter(Boolean).map((token) => SYNONYMS[token] ?? token).filter(Boolean).map(depluralize).sort().join(" ");
|
|
2581
2581
|
}
|
|
2582
|
-
function isMetricDimension$2(dim) {
|
|
2583
|
-
return [
|
|
2584
|
-
"clicks",
|
|
2585
|
-
"impressions",
|
|
2586
|
-
"ctr",
|
|
2587
|
-
"position"
|
|
2588
|
-
].includes(dim);
|
|
2589
|
-
}
|
|
2590
|
-
const ENGINE_SOURCE_CAPABILITIES = {
|
|
2591
|
-
regex: true,
|
|
2592
|
-
multiDataset: false,
|
|
2593
|
-
comparisonJoin: false,
|
|
2594
|
-
windowTotals: false,
|
|
2595
|
-
fileSets: true,
|
|
2596
|
-
localSource: true
|
|
2597
|
-
};
|
|
2598
|
-
function createEngineQuerySource(options) {
|
|
2599
|
-
const { engine, ctx } = options;
|
|
2600
|
-
return {
|
|
2601
|
-
name: "engine",
|
|
2602
|
-
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
2603
|
-
async queryRows(state) {
|
|
2604
|
-
const filterDims = getFilterDimensions(state.filter, isMetricDimension$2);
|
|
2605
|
-
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
2606
|
-
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
2607
|
-
return (await engine.query(ctx, state)).rows;
|
|
2608
|
-
},
|
|
2609
|
-
async executeSql(sql, params, opts) {
|
|
2610
|
-
const fileSets = opts?.fileSets;
|
|
2611
|
-
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
2612
|
-
const { rows } = await engine.runSQL({
|
|
2613
|
-
ctx,
|
|
2614
|
-
table: fileSets.FILES.table,
|
|
2615
|
-
fileSets,
|
|
2616
|
-
sql,
|
|
2617
|
-
params: params ?? []
|
|
2618
|
-
});
|
|
2619
|
-
return rows;
|
|
2620
|
-
}
|
|
2621
|
-
};
|
|
2622
|
-
}
|
|
2623
|
-
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
2624
|
-
return runAnalyzerFromSource(createEngineQuerySource({
|
|
2625
|
-
engine: deps.engine,
|
|
2626
|
-
ctx
|
|
2627
|
-
}), params, registry);
|
|
2628
|
-
}
|
|
2629
2582
|
async function collectRows(gen) {
|
|
2630
2583
|
const out = [];
|
|
2631
2584
|
for await (const batch of gen) out.push(...batch);
|
|
@@ -2637,11 +2590,11 @@ const METRIC_NAMES = [
|
|
|
2637
2590
|
"ctr",
|
|
2638
2591
|
"position"
|
|
2639
2592
|
];
|
|
2640
|
-
function isMetricDimension$
|
|
2593
|
+
function isMetricDimension$2(dim) {
|
|
2641
2594
|
return METRIC_NAMES.includes(dim);
|
|
2642
2595
|
}
|
|
2643
2596
|
function applyBuilderStatePostProcessing(rows, state) {
|
|
2644
|
-
const dimensionFilters = getDimensionFilters(state.filter, isMetricDimension$
|
|
2597
|
+
const dimensionFilters = getDimensionFilters(state.filter, isMetricDimension$2);
|
|
2645
2598
|
const metricFilters = extractMetricFilters(state.filter);
|
|
2646
2599
|
const specialFilters = extractSpecialOperatorFilters(state.filter);
|
|
2647
2600
|
const ordered = [...rows.filter((row) => {
|
|
@@ -2668,7 +2621,7 @@ const GSC_API_CAPABILITIES = {
|
|
|
2668
2621
|
comparisonJoin: false,
|
|
2669
2622
|
windowTotals: false
|
|
2670
2623
|
};
|
|
2671
|
-
function isMetricDimension(dim) {
|
|
2624
|
+
function isMetricDimension$1(dim) {
|
|
2672
2625
|
return [
|
|
2673
2626
|
"clicks",
|
|
2674
2627
|
"impressions",
|
|
@@ -2686,12 +2639,59 @@ function createGscApiQuerySource(options) {
|
|
|
2686
2639
|
capabilities: GSC_API_CAPABILITIES,
|
|
2687
2640
|
async queryRows(state) {
|
|
2688
2641
|
buildLogicalPlan(state, GSC_API_CAPABILITIES);
|
|
2689
|
-
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
2642
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension$1);
|
|
2690
2643
|
assertDimensionsSupported([...state.dimensions, ...filterDims], "api", "gsc-api query source");
|
|
2691
2644
|
return applyBuilderStatePostProcessing(await collectRows(client.query(siteUrl, builderFromState(state))), state);
|
|
2692
2645
|
}
|
|
2693
2646
|
};
|
|
2694
2647
|
}
|
|
2648
|
+
function isMetricDimension(dim) {
|
|
2649
|
+
return [
|
|
2650
|
+
"clicks",
|
|
2651
|
+
"impressions",
|
|
2652
|
+
"ctr",
|
|
2653
|
+
"position"
|
|
2654
|
+
].includes(dim);
|
|
2655
|
+
}
|
|
2656
|
+
const ENGINE_SOURCE_CAPABILITIES = {
|
|
2657
|
+
regex: true,
|
|
2658
|
+
multiDataset: false,
|
|
2659
|
+
comparisonJoin: false,
|
|
2660
|
+
windowTotals: false,
|
|
2661
|
+
fileSets: true,
|
|
2662
|
+
localSource: true
|
|
2663
|
+
};
|
|
2664
|
+
function createEngineQuerySource(options) {
|
|
2665
|
+
const { engine, ctx } = options;
|
|
2666
|
+
return {
|
|
2667
|
+
name: "engine",
|
|
2668
|
+
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
2669
|
+
async queryRows(state) {
|
|
2670
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
2671
|
+
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
2672
|
+
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
2673
|
+
return (await engine.query(ctx, state)).rows;
|
|
2674
|
+
},
|
|
2675
|
+
async executeSql(sql, params, opts) {
|
|
2676
|
+
const fileSets = opts?.fileSets;
|
|
2677
|
+
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
2678
|
+
const { rows } = await engine.runSQL({
|
|
2679
|
+
ctx,
|
|
2680
|
+
table: fileSets.FILES.table,
|
|
2681
|
+
fileSets,
|
|
2682
|
+
sql,
|
|
2683
|
+
params: params ?? []
|
|
2684
|
+
});
|
|
2685
|
+
return rows;
|
|
2686
|
+
}
|
|
2687
|
+
};
|
|
2688
|
+
}
|
|
2689
|
+
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
2690
|
+
return runAnalyzerFromSource(createEngineQuerySource({
|
|
2691
|
+
engine: deps.engine,
|
|
2692
|
+
ctx
|
|
2693
|
+
}), params, registry);
|
|
2694
|
+
}
|
|
2695
2695
|
const IN_MEMORY_DEFAULT_CAPABILITIES = {
|
|
2696
2696
|
regex: true,
|
|
2697
2697
|
multiDataset: true,
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
type MigrationPhase = 'd1' | 'dual' | 'r2';
|
|
2
|
+
type MigrationReadFrom = 'd1' | 'r2';
|
|
3
|
+
interface RoutingInputs {
|
|
4
|
+
userPhase: MigrationPhase | string | null | undefined;
|
|
5
|
+
userReadFrom: MigrationReadFrom | string | null | undefined;
|
|
6
|
+
sitePhase?: MigrationPhase | string | null;
|
|
7
|
+
siteReadFrom?: MigrationReadFrom | string | null;
|
|
8
|
+
}
|
|
9
|
+
interface RoutingEnv {
|
|
10
|
+
ANALYTICS_FORCE_D1?: string;
|
|
11
|
+
R2_READS_ENABLED?: string;
|
|
12
|
+
}
|
|
13
|
+
declare function resolvePhase(inputs: Pick<RoutingInputs, 'userPhase' | 'sitePhase'>): MigrationPhase;
|
|
14
|
+
declare function resolveReadFrom(inputs: Pick<RoutingInputs, 'userReadFrom' | 'siteReadFrom'>): MigrationReadFrom;
|
|
15
|
+
declare function forceD1(env: Pick<RoutingEnv, 'ANALYTICS_FORCE_D1'>): boolean;
|
|
16
|
+
declare function r2ReadsEnabled(env: Pick<RoutingEnv, 'R2_READS_ENABLED'>): boolean;
|
|
17
|
+
declare function shouldDualWrite(inputs: RoutingInputs, env: Pick<RoutingEnv, 'ANALYTICS_FORCE_D1'>): boolean;
|
|
18
|
+
type AnalyticsTable = 'pages' | 'keywords' | 'countries' | 'devices' | 'page_keywords';
|
|
19
|
+
declare function isAnalyticsTable(table: string): table is AnalyticsTable;
|
|
20
|
+
declare function shouldWriteToD1(inputs: RoutingInputs, env: Pick<RoutingEnv, 'ANALYTICS_FORCE_D1'>, table: string): boolean;
|
|
21
|
+
declare function shouldReadFromR2(inputs: RoutingInputs, env: Pick<RoutingEnv, 'ANALYTICS_FORCE_D1' | 'R2_READS_ENABLED'>): boolean;
|
|
22
|
+
declare function isValidReadFromForPhase(phase: MigrationPhase, readFrom: MigrationReadFrom): boolean;
|
|
23
|
+
export { AnalyticsTable, MigrationPhase, MigrationReadFrom, RoutingEnv, RoutingInputs, forceD1, isAnalyticsTable, isValidReadFromForPhase, r2ReadsEnabled, resolvePhase, resolveReadFrom, shouldDualWrite, shouldReadFromR2, shouldWriteToD1 };
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
function isPhase(v) {
|
|
2
|
+
return v === "d1" || v === "dual" || v === "r2";
|
|
3
|
+
}
|
|
4
|
+
function isReadFrom(v) {
|
|
5
|
+
return v === "d1" || v === "r2";
|
|
6
|
+
}
|
|
7
|
+
function resolvePhase(inputs) {
|
|
8
|
+
if (isPhase(inputs.sitePhase)) return inputs.sitePhase;
|
|
9
|
+
if (isPhase(inputs.userPhase)) return inputs.userPhase;
|
|
10
|
+
return "d1";
|
|
11
|
+
}
|
|
12
|
+
function resolveReadFrom(inputs) {
|
|
13
|
+
if (isReadFrom(inputs.siteReadFrom)) return inputs.siteReadFrom;
|
|
14
|
+
if (isReadFrom(inputs.userReadFrom)) return inputs.userReadFrom;
|
|
15
|
+
return "d1";
|
|
16
|
+
}
|
|
17
|
+
function forceD1(env) {
|
|
18
|
+
return env.ANALYTICS_FORCE_D1 === "1";
|
|
19
|
+
}
|
|
20
|
+
function r2ReadsEnabled(env) {
|
|
21
|
+
return env.R2_READS_ENABLED === "1";
|
|
22
|
+
}
|
|
23
|
+
function shouldDualWrite(inputs, env) {
|
|
24
|
+
if (forceD1(env)) return false;
|
|
25
|
+
const phase = resolvePhase(inputs);
|
|
26
|
+
return phase === "dual" || phase === "r2";
|
|
27
|
+
}
|
|
28
|
+
const ANALYTICS_TABLES = new Set([
|
|
29
|
+
"pages",
|
|
30
|
+
"keywords",
|
|
31
|
+
"countries",
|
|
32
|
+
"devices",
|
|
33
|
+
"page_keywords"
|
|
34
|
+
]);
|
|
35
|
+
function isAnalyticsTable(table) {
|
|
36
|
+
return ANALYTICS_TABLES.has(table);
|
|
37
|
+
}
|
|
38
|
+
function shouldWriteToD1(inputs, env, table) {
|
|
39
|
+
if (forceD1(env)) return true;
|
|
40
|
+
if (!isAnalyticsTable(table)) return true;
|
|
41
|
+
return resolvePhase(inputs) !== "r2";
|
|
42
|
+
}
|
|
43
|
+
function shouldReadFromR2(inputs, env) {
|
|
44
|
+
if (forceD1(env)) return false;
|
|
45
|
+
if (!r2ReadsEnabled(env)) return false;
|
|
46
|
+
if (resolvePhase(inputs) === "d1") return false;
|
|
47
|
+
return resolveReadFrom(inputs) === "r2";
|
|
48
|
+
}
|
|
49
|
+
function isValidReadFromForPhase(phase, readFrom) {
|
|
50
|
+
if (readFrom === "r2" && phase === "d1") return false;
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
export { forceD1, isAnalyticsTable, isValidReadFromForPhase, r2ReadsEnabled, resolvePhase, resolveReadFrom, shouldDualWrite, shouldReadFromR2, shouldWriteToD1 };
|
package/dist/source/index.d.mts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { BrowserQueryRunner, createEngine as createBrowserQuerySource } from "@gscdump/engine-wasm";
|
|
2
|
-
import { AnalysisQuerySource, AnalysisQuerySource as AnalysisQuerySource$1, FileSet, QueryRow, QueryRow as QueryRow$1, RowQuerySource, SourceCapabilities, SqlQuerySource, isSqlQuerySource } from "@gscdump/engine/resolver";
|
|
3
|
-
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
4
2
|
import { BuilderState, Column, Dimension } from "gscdump/query";
|
|
5
3
|
import { GoogleSearchConsoleClient } from "gscdump";
|
|
4
|
+
import { AnalysisQuerySource, AnalysisQuerySource as AnalysisQuerySource$1, FileSet, QueryRow, QueryRow as QueryRow$1, RowQuerySource, SourceCapabilities, SqlQuerySource, isSqlQuerySource } from "@gscdump/engine/resolver";
|
|
5
|
+
import { PlannerCapabilities } from "gscdump/query/plan";
|
|
6
6
|
import { EngineConfig, SqliteQueryExecutor, createEngine as createSqliteQuerySource } from "@gscdump/engine-sqlite";
|
|
7
7
|
import { Row, StorageEngine, TenantCtx } from "@gscdump/engine/contracts";
|
|
8
8
|
import { AnalysisParams, AnalysisResult } from "gscdump/contracts";
|
|
@@ -121,6 +121,15 @@ declare class AnalyzerCapabilityError extends Error {
|
|
|
121
121
|
constructor(tool: string, missing: readonly Capability[]);
|
|
122
122
|
}
|
|
123
123
|
declare function analyzeFromSource(source: AnalysisQuerySource, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
|
|
124
|
+
interface CompositeSourceOptions {
|
|
125
|
+
engine: SqlQuerySource;
|
|
126
|
+
live: AnalysisQuerySource;
|
|
127
|
+
site: {
|
|
128
|
+
oldestDateSynced: string | null;
|
|
129
|
+
newestDateSynced: string | null;
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
declare function createCompositeSource(opts: CompositeSourceOptions): SqlQuerySource;
|
|
124
133
|
/**
|
|
125
134
|
* Capabilities the engine query path honors. Matches what the DuckDB compiler
|
|
126
135
|
* passes to {@link buildLogicalPlan} (see `gscdump/analytics/compiler`): regex
|
|
@@ -209,6 +218,17 @@ interface InMemoryQuerySourceOptions {
|
|
|
209
218
|
capabilities?: PlannerCapabilities;
|
|
210
219
|
}
|
|
211
220
|
declare function createInMemoryQuerySource(options: InMemoryQuerySourceOptions): RowQuerySource;
|
|
221
|
+
declare function canProxyToGsc(state: BuilderState): boolean;
|
|
222
|
+
interface CreateLiveGscSourceOptions {
|
|
223
|
+
/** GSC property URL (e.g. `sc-domain:example.com` or `https://example.com/`). */
|
|
224
|
+
siteUrl: string;
|
|
225
|
+
/**
|
|
226
|
+
* Returns a valid GSC access token. Called lazily on first query so refresh
|
|
227
|
+
* cost is paid only when the source actually runs. Host owns refresh logic.
|
|
228
|
+
*/
|
|
229
|
+
getAccessToken: () => Promise<string>;
|
|
230
|
+
}
|
|
231
|
+
declare function createLiveGscSource(opts: CreateLiveGscSourceOptions): AnalysisQuerySource;
|
|
212
232
|
interface BrandSegmentationOptions {
|
|
213
233
|
/** Brand terms to match against keywords (case-insensitive) */
|
|
214
234
|
brandTerms: string[];
|
|
@@ -424,4 +444,4 @@ declare function analyzeSeasonalityFromSource(source: AnalysisQuerySource, perio
|
|
|
424
444
|
declare function analyzeDecayFromSource(source: AnalysisQuerySource, periods: ComparisonPeriod, options?: DecayOptions): Promise<DecayResult[]>;
|
|
425
445
|
declare function analyzeMoversFromSource(source: AnalysisQuerySource, periods: ComparisonPeriod, options?: MoversOptions): Promise<MoversResult>;
|
|
426
446
|
type SqliteQuerySourceOptions = EngineConfig;
|
|
427
|
-
export { type AnalysisQuerySource, AnalyzerCapabilityError, type BrowserQueryRunner, type ComparisonQueryResult, ENGINE_QUERY_CAPABILITIES, type EngineQuerySourceOptions, type FetchTopNOptions, GSC_API_CAPABILITIES, type GscApiQuerySourceOptions, type GscDailyRow, type GscRange, type GscTopNRow, IN_MEMORY_DEFAULT_CAPABILITIES, type InMemoryQuerySourceOptions, type QueryDimension, type QueryOptions, type QueryResult, type QueryRow, type RowQuerySource, type SourceCapabilities, type SqlQuerySource, type SqliteQueryExecutor, type SqliteQuerySourceOptions, type TypedQuery, analyzeBrandSegmentationFromSource, analyzeClusteringFromSource, analyzeDecayFromSource, analyzeFromSource, analyzeKeywordConcentrationFromSource, analyzeMoversFromSource, analyzeOpportunityFromSource, analyzePageConcentrationFromSource, analyzeSeasonalityFromSource, analyzeStrikingDistanceFromSource, collectRows as collectGscRows, createBrowserQuerySource, createEngineQuerySource, createGscApiQuerySource, createInMemoryQuerySource, createSqliteQuerySource, fetchGscDaily, fetchGscTopN, isSqlQuerySource, queryAnalyticsFromSource, queryComparisonFromSource, queryComparisonRows, queryRows, runAnalyzerWithEngine, typedQuery };
|
|
447
|
+
export { type AnalysisQuerySource, AnalyzerCapabilityError, type BrowserQueryRunner, type ComparisonQueryResult, type CompositeSourceOptions, type CreateLiveGscSourceOptions, ENGINE_QUERY_CAPABILITIES, type EngineQuerySourceOptions, type FetchTopNOptions, GSC_API_CAPABILITIES, type GscApiQuerySourceOptions, type GscDailyRow, type GscRange, type GscTopNRow, IN_MEMORY_DEFAULT_CAPABILITIES, type InMemoryQuerySourceOptions, type QueryDimension, type QueryOptions, type QueryResult, type QueryRow, type RowQuerySource, type SourceCapabilities, type SqlQuerySource, type SqliteQueryExecutor, type SqliteQuerySourceOptions, type TypedQuery, analyzeBrandSegmentationFromSource, analyzeClusteringFromSource, analyzeDecayFromSource, analyzeFromSource, analyzeKeywordConcentrationFromSource, analyzeMoversFromSource, analyzeOpportunityFromSource, analyzePageConcentrationFromSource, analyzeSeasonalityFromSource, analyzeStrikingDistanceFromSource, canProxyToGsc, collectRows as collectGscRows, createBrowserQuerySource, createCompositeSource, createEngineQuerySource, createGscApiQuerySource, createInMemoryQuerySource, createLiveGscSource, createSqliteQuerySource, fetchGscDaily, fetchGscTopN, isSqlQuerySource, queryAnalyticsFromSource, queryComparisonFromSource, queryComparisonRows, queryRows, runAnalyzerWithEngine, typedQuery };
|
package/dist/source/index.mjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { createEngine as createBrowserQuerySource } from "@gscdump/engine-wasm";
|
|
2
|
+
import { between, clicks, date, extractDateRange, extractMetricFilters, extractSpecialOperatorFilters, gsc, page, query } from "gscdump/query";
|
|
3
|
+
import { daysAgo, googleSearchConsole } from "gscdump";
|
|
2
4
|
import { assertDimensionsSupported, getDimensionFilters, getFilterDimensions, isSqlQuerySource, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, metricValue } from "@gscdump/engine/resolver";
|
|
3
5
|
import { buildLogicalPlan } from "gscdump/query/plan";
|
|
4
|
-
import { between, clicks, date, extractMetricFilters, extractSpecialOperatorFilters, gsc, page, query } from "gscdump/query";
|
|
5
6
|
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
6
7
|
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
7
|
-
import { daysAgo } from "gscdump";
|
|
8
8
|
import { createEngine as createSqliteQuerySource } from "@gscdump/engine-sqlite";
|
|
9
9
|
var AnalyzerCapabilityError = class extends Error {
|
|
10
10
|
constructor(tool, missing) {
|
|
@@ -83,56 +83,6 @@ async function runSqlPlanAgainstSource(source, analyzer, plan, params) {
|
|
|
83
83
|
async function analyzeFromSource(source, params, registry) {
|
|
84
84
|
return runAnalyzerFromSource(source, params, registry);
|
|
85
85
|
}
|
|
86
|
-
function isMetricDimension$2(dim) {
|
|
87
|
-
return [
|
|
88
|
-
"clicks",
|
|
89
|
-
"impressions",
|
|
90
|
-
"ctr",
|
|
91
|
-
"position"
|
|
92
|
-
].includes(dim);
|
|
93
|
-
}
|
|
94
|
-
const ENGINE_QUERY_CAPABILITIES = {
|
|
95
|
-
regex: true,
|
|
96
|
-
multiDataset: false,
|
|
97
|
-
comparisonJoin: false,
|
|
98
|
-
windowTotals: false
|
|
99
|
-
};
|
|
100
|
-
const ENGINE_SOURCE_CAPABILITIES = {
|
|
101
|
-
...ENGINE_QUERY_CAPABILITIES,
|
|
102
|
-
fileSets: true,
|
|
103
|
-
localSource: true
|
|
104
|
-
};
|
|
105
|
-
function createEngineQuerySource(options) {
|
|
106
|
-
const { engine, ctx } = options;
|
|
107
|
-
return {
|
|
108
|
-
name: "engine",
|
|
109
|
-
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
110
|
-
async queryRows(state) {
|
|
111
|
-
const filterDims = getFilterDimensions(state.filter, isMetricDimension$2);
|
|
112
|
-
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
113
|
-
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
114
|
-
return (await engine.query(ctx, state)).rows;
|
|
115
|
-
},
|
|
116
|
-
async executeSql(sql, params, opts) {
|
|
117
|
-
const fileSets = opts?.fileSets;
|
|
118
|
-
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
119
|
-
const { rows } = await engine.runSQL({
|
|
120
|
-
ctx,
|
|
121
|
-
table: fileSets.FILES.table,
|
|
122
|
-
fileSets,
|
|
123
|
-
sql,
|
|
124
|
-
params: params ?? []
|
|
125
|
-
});
|
|
126
|
-
return rows;
|
|
127
|
-
}
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
131
|
-
return runAnalyzerFromSource(createEngineQuerySource({
|
|
132
|
-
engine: deps.engine,
|
|
133
|
-
ctx
|
|
134
|
-
}), params, registry);
|
|
135
|
-
}
|
|
136
86
|
async function collectRows(gen) {
|
|
137
87
|
const out = [];
|
|
138
88
|
for await (const batch of gen) out.push(...batch);
|
|
@@ -181,11 +131,11 @@ const METRIC_NAMES = [
|
|
|
181
131
|
"ctr",
|
|
182
132
|
"position"
|
|
183
133
|
];
|
|
184
|
-
function isMetricDimension$
|
|
134
|
+
function isMetricDimension$2(dim) {
|
|
185
135
|
return METRIC_NAMES.includes(dim);
|
|
186
136
|
}
|
|
187
137
|
function applyBuilderStatePostProcessing(rows, state) {
|
|
188
|
-
const dimensionFilters = getDimensionFilters(state.filter, isMetricDimension$
|
|
138
|
+
const dimensionFilters = getDimensionFilters(state.filter, isMetricDimension$2);
|
|
189
139
|
const metricFilters = extractMetricFilters(state.filter);
|
|
190
140
|
const specialFilters = extractSpecialOperatorFilters(state.filter);
|
|
191
141
|
const ordered = [...rows.filter((row) => {
|
|
@@ -212,7 +162,7 @@ const GSC_API_CAPABILITIES = {
|
|
|
212
162
|
comparisonJoin: false,
|
|
213
163
|
windowTotals: false
|
|
214
164
|
};
|
|
215
|
-
function isMetricDimension(dim) {
|
|
165
|
+
function isMetricDimension$1(dim) {
|
|
216
166
|
return [
|
|
217
167
|
"clicks",
|
|
218
168
|
"impressions",
|
|
@@ -230,12 +180,107 @@ function createGscApiQuerySource(options) {
|
|
|
230
180
|
capabilities: GSC_API_CAPABILITIES,
|
|
231
181
|
async queryRows(state) {
|
|
232
182
|
buildLogicalPlan(state, GSC_API_CAPABILITIES);
|
|
233
|
-
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
183
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension$1);
|
|
234
184
|
assertDimensionsSupported([...state.dimensions, ...filterDims], "api", "gsc-api query source");
|
|
235
185
|
return applyBuilderStatePostProcessing(await collectRows(client.query(siteUrl, builderFromState(state))), state);
|
|
236
186
|
}
|
|
237
187
|
};
|
|
238
188
|
}
|
|
189
|
+
const PRO_ONLY_DIMENSIONS = new Set(["queryCanonical", "page_keywords"]);
|
|
190
|
+
function canProxyToGsc(state) {
|
|
191
|
+
if (state.dimensions.some((d) => PRO_ONLY_DIMENSIONS.has(d))) return false;
|
|
192
|
+
if (extractMetricFilters(state.filter).length > 0) return false;
|
|
193
|
+
if (extractSpecialOperatorFilters(state.filter).length > 0) return false;
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
function createLiveGscSource(opts) {
|
|
197
|
+
let clientPromise = null;
|
|
198
|
+
function getClient() {
|
|
199
|
+
if (!clientPromise) clientPromise = opts.getAccessToken().then((accessToken) => googleSearchConsole({ accessToken }));
|
|
200
|
+
return clientPromise;
|
|
201
|
+
}
|
|
202
|
+
return {
|
|
203
|
+
name: "gsc-api",
|
|
204
|
+
capabilities: {
|
|
205
|
+
regex: true,
|
|
206
|
+
multiDataset: false,
|
|
207
|
+
comparisonJoin: false,
|
|
208
|
+
windowTotals: false
|
|
209
|
+
},
|
|
210
|
+
async queryRows(state) {
|
|
211
|
+
return createGscApiQuerySource({
|
|
212
|
+
client: await getClient(),
|
|
213
|
+
siteUrl: opts.siteUrl
|
|
214
|
+
}).queryRows(state);
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
function createCompositeSource(opts) {
|
|
219
|
+
const { engine, live, site } = opts;
|
|
220
|
+
function rangeCovered(state) {
|
|
221
|
+
const { startDate, endDate } = extractDateRange(state.filter);
|
|
222
|
+
return !!(startDate && endDate && site.oldestDateSynced && site.newestDateSynced && startDate >= site.oldestDateSynced && endDate <= site.newestDateSynced);
|
|
223
|
+
}
|
|
224
|
+
return {
|
|
225
|
+
name: "composite-engine-live",
|
|
226
|
+
capabilities: engine.capabilities,
|
|
227
|
+
async queryRows(state) {
|
|
228
|
+
if (!rangeCovered(state) && canProxyToGsc(state)) return live.queryRows(state);
|
|
229
|
+
return engine.queryRows(state);
|
|
230
|
+
},
|
|
231
|
+
executeSql: engine.executeSql
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
function isMetricDimension(dim) {
|
|
235
|
+
return [
|
|
236
|
+
"clicks",
|
|
237
|
+
"impressions",
|
|
238
|
+
"ctr",
|
|
239
|
+
"position"
|
|
240
|
+
].includes(dim);
|
|
241
|
+
}
|
|
242
|
+
const ENGINE_QUERY_CAPABILITIES = {
|
|
243
|
+
regex: true,
|
|
244
|
+
multiDataset: false,
|
|
245
|
+
comparisonJoin: false,
|
|
246
|
+
windowTotals: false
|
|
247
|
+
};
|
|
248
|
+
const ENGINE_SOURCE_CAPABILITIES = {
|
|
249
|
+
...ENGINE_QUERY_CAPABILITIES,
|
|
250
|
+
fileSets: true,
|
|
251
|
+
localSource: true
|
|
252
|
+
};
|
|
253
|
+
function createEngineQuerySource(options) {
|
|
254
|
+
const { engine, ctx } = options;
|
|
255
|
+
return {
|
|
256
|
+
name: "engine",
|
|
257
|
+
capabilities: ENGINE_SOURCE_CAPABILITIES,
|
|
258
|
+
async queryRows(state) {
|
|
259
|
+
const filterDims = getFilterDimensions(state.filter, isMetricDimension);
|
|
260
|
+
assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
|
|
261
|
+
if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
|
|
262
|
+
return (await engine.query(ctx, state)).rows;
|
|
263
|
+
},
|
|
264
|
+
async executeSql(sql, params, opts) {
|
|
265
|
+
const fileSets = opts?.fileSets;
|
|
266
|
+
if (!fileSets?.FILES) throw new Error("engine query source: executeSql requires opts.fileSets with a FILES entry");
|
|
267
|
+
const { rows } = await engine.runSQL({
|
|
268
|
+
ctx,
|
|
269
|
+
table: fileSets.FILES.table,
|
|
270
|
+
fileSets,
|
|
271
|
+
sql,
|
|
272
|
+
params: params ?? []
|
|
273
|
+
});
|
|
274
|
+
return rows;
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
async function runAnalyzerWithEngine(deps, ctx, params, registry) {
|
|
279
|
+
return runAnalyzerFromSource(createEngineQuerySource({
|
|
280
|
+
engine: deps.engine,
|
|
281
|
+
ctx
|
|
282
|
+
}), params, registry);
|
|
283
|
+
}
|
|
239
284
|
const IN_MEMORY_DEFAULT_CAPABILITIES = {
|
|
240
285
|
regex: true,
|
|
241
286
|
multiDataset: true,
|
|
@@ -1862,4 +1907,4 @@ async function analyzeDecayFromSource(source, periods, options) {
|
|
|
1862
1907
|
async function analyzeMoversFromSource(source, periods, options) {
|
|
1863
1908
|
return runPortableAnalyzer(source, PORTABLE_ANALYZERS.movers, periods, options);
|
|
1864
1909
|
}
|
|
1865
|
-
export { AnalyzerCapabilityError, ENGINE_QUERY_CAPABILITIES, GSC_API_CAPABILITIES, IN_MEMORY_DEFAULT_CAPABILITIES, analyzeBrandSegmentationFromSource, analyzeClusteringFromSource, analyzeDecayFromSource, analyzeFromSource, analyzeKeywordConcentrationFromSource, analyzeMoversFromSource, analyzeOpportunityFromSource, analyzePageConcentrationFromSource, analyzeSeasonalityFromSource, analyzeStrikingDistanceFromSource, collectRows as collectGscRows, createBrowserQuerySource, createEngineQuerySource, createGscApiQuerySource, createInMemoryQuerySource, createSqliteQuerySource, fetchGscDaily, fetchGscTopN, isSqlQuerySource, queryAnalyticsFromSource, queryComparisonFromSource, queryComparisonRows, queryRows, runAnalyzerWithEngine, typedQuery };
|
|
1910
|
+
export { AnalyzerCapabilityError, ENGINE_QUERY_CAPABILITIES, GSC_API_CAPABILITIES, IN_MEMORY_DEFAULT_CAPABILITIES, analyzeBrandSegmentationFromSource, analyzeClusteringFromSource, analyzeDecayFromSource, analyzeFromSource, analyzeKeywordConcentrationFromSource, analyzeMoversFromSource, analyzeOpportunityFromSource, analyzePageConcentrationFromSource, analyzeSeasonalityFromSource, analyzeStrikingDistanceFromSource, canProxyToGsc, collectRows as collectGscRows, createBrowserQuerySource, createCompositeSource, createEngineQuerySource, createGscApiQuerySource, createInMemoryQuerySource, createLiveGscSource, createSqliteQuerySource, fetchGscDaily, fetchGscTopN, isSqlQuerySource, queryAnalyticsFromSource, queryComparisonFromSource, queryComparisonRows, queryRows, runAnalyzerWithEngine, typedQuery };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/analysis",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.7.1",
|
|
5
5
|
"description": "GSC analyzers — striking-distance, opportunity, movers, decay, brand, clustering, concentration, seasonality. Pure row-based + DuckDB-native.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -48,6 +48,10 @@
|
|
|
48
48
|
"./period": {
|
|
49
49
|
"types": "./dist/period/index.d.mts",
|
|
50
50
|
"import": "./dist/period/index.mjs"
|
|
51
|
+
},
|
|
52
|
+
"./routing": {
|
|
53
|
+
"types": "./dist/routing/index.d.mts",
|
|
54
|
+
"import": "./dist/routing/index.mjs"
|
|
51
55
|
}
|
|
52
56
|
},
|
|
53
57
|
"main": "./dist/index.mjs",
|
|
@@ -68,11 +72,11 @@
|
|
|
68
72
|
},
|
|
69
73
|
"dependencies": {
|
|
70
74
|
"drizzle-orm": "^0.45.2",
|
|
71
|
-
"@gscdump/engine
|
|
72
|
-
"@gscdump/engine": "0.
|
|
73
|
-
"@gscdump/engine-
|
|
74
|
-
"
|
|
75
|
-
"gscdump": "0.
|
|
75
|
+
"@gscdump/engine": "0.7.1",
|
|
76
|
+
"@gscdump/engine-sqlite": "0.7.1",
|
|
77
|
+
"@gscdump/engine-wasm": "0.7.1",
|
|
78
|
+
"gscdump": "0.7.1",
|
|
79
|
+
"@gscdump/engine-duckdb-node": "0.7.1"
|
|
76
80
|
},
|
|
77
81
|
"devDependencies": {
|
|
78
82
|
"vitest": "^4.1.5"
|