@gscdump/cli 3.8.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -30
- package/bin/gscdump.mjs +21 -1
- package/dist/analysis-local.mjs +172 -53
- package/dist/auth-state.mjs +21 -6
- package/dist/auth.mjs +27 -18
- package/dist/bing-data.mjs +6 -2
- package/dist/bing-hosted.mjs +1 -1
- package/dist/cli-args.mjs +8 -0
- package/dist/cli.d.mts +5 -1
- package/dist/cli.mjs +41 -10
- package/dist/cloud-google.mjs +8 -1
- package/dist/command-meta.mjs +4 -4
- package/dist/command-registry.mjs +86 -10
- package/dist/commands/analyze.mjs +84 -43
- package/dist/commands/auth.mjs +46 -37
- package/dist/commands/bing.mjs +3 -3
- package/dist/commands/compact.mjs +12 -8
- package/dist/commands/config.mjs +14 -14
- package/dist/commands/doctor.mjs +26 -26
- package/dist/commands/dump.mjs +203 -105
- package/dist/commands/entities.mjs +10 -113
- package/dist/commands/gc.mjs +4 -3
- package/dist/commands/indexing-urls.mjs +185 -0
- package/dist/commands/indexing.mjs +12 -8
- package/dist/commands/init.mjs +77 -18
- package/dist/commands/inspect.mjs +97 -110
- package/dist/commands/mcp.mjs +37 -57
- package/dist/commands/papercut.mjs +1 -1
- package/dist/commands/profile.mjs +9 -17
- package/dist/commands/query.mjs +219 -210
- package/dist/commands/report.mjs +68 -85
- package/dist/commands/rollups.mjs +9 -6
- package/dist/commands/sitemaps.mjs +51 -104
- package/dist/commands/sites.mjs +40 -24
- package/dist/commands/stats.mjs +14 -17
- package/dist/commands/store-purge.mjs +9 -5
- package/dist/commands/store.mjs +0 -12
- package/dist/commands/sync.mjs +774 -296
- package/dist/config.mjs +7 -4
- package/dist/context.mjs +51 -21
- package/dist/coverage.mjs +251 -0
- package/dist/dump-bing.mjs +71 -0
- package/dist/dump-writers.mjs +246 -0
- package/dist/error-handler.mjs +108 -38
- package/dist/filters.mjs +125 -0
- package/dist/hosted-site.mjs +91 -0
- package/dist/inspect-urls.mjs +76 -0
- package/dist/inspection-record.mjs +93 -0
- package/dist/local-entities.mjs +658 -0
- package/dist/mcp/errors.mjs +71 -18
- package/dist/mcp/handlers/diagnostics.mjs +15 -12
- package/dist/mcp/handlers/reports.mjs +33 -42
- package/dist/mcp/server/index.mjs +80 -38
- package/dist/mcp/types.mjs +7 -7
- package/dist/package.mjs +1 -1
- package/dist/quota-ledger.mjs +370 -0
- package/dist/render/analysis.mjs +8 -1
- package/dist/render/report.mjs +8 -1
- package/dist/request-pacer.mjs +29 -0
- package/dist/route.mjs +295 -0
- package/dist/sitemap.mjs +4 -0
- package/dist/sql-views.mjs +117 -0
- package/dist/store-sites.mjs +134 -0
- package/dist/sync-plan.mjs +129 -0
- package/dist/sync-run.mjs +96 -0
- package/dist/table-sources.mjs +57 -0
- package/dist/token-info.mjs +54 -0
- package/dist/utils.mjs +28 -19
- package/dist/window.mjs +159 -0
- package/package.json +17 -14
- package/skills/gscdump/SKILL.md +167 -37
- package/dist/commands/export.mjs +0 -76
- package/dist/native-duckdb.mjs +0 -46
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { exportColumns, readParquetSql, sourceSelectSql, sqlString } from "./table-sources.mjs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import { DuckDBInstance } from "@duckdb/node-api";
|
|
7
|
+
import { encodeRowsToParquetFlex } from "@gscdump/engine/hyparquet";
|
|
8
|
+
const DUMP_FORMATS = [
|
|
9
|
+
"parquet",
|
|
10
|
+
"csv",
|
|
11
|
+
"json",
|
|
12
|
+
"ndjson",
|
|
13
|
+
"sqlite",
|
|
14
|
+
"duckdb"
|
|
15
|
+
];
|
|
16
|
+
function isDumpFormat(value) {
|
|
17
|
+
return DUMP_FORMATS.includes(value);
|
|
18
|
+
}
|
|
19
|
+
function siteDirName(siteUrl) {
|
|
20
|
+
return siteUrl.replace(/[^a-z0-9]+/gi, "_");
|
|
21
|
+
}
|
|
22
|
+
function createSiteDirNamer() {
|
|
23
|
+
const dirBySite = /* @__PURE__ */ new Map();
|
|
24
|
+
return (siteUrl) => {
|
|
25
|
+
const known = dirBySite.get(siteUrl);
|
|
26
|
+
if (known !== void 0) return known;
|
|
27
|
+
const base = siteDirName(siteUrl);
|
|
28
|
+
const dir = [...dirBySite.values()].includes(base) ? `${base}_${createHash("sha256").update(siteUrl).digest("hex").slice(0, 8)}` : base;
|
|
29
|
+
dirBySite.set(siteUrl, dir);
|
|
30
|
+
return dir;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function databasePath(outDir, format) {
|
|
34
|
+
return path.join(outDir, `gscdump.${format}`);
|
|
35
|
+
}
|
|
36
|
+
const COPY_OPTIONS = {
|
|
37
|
+
parquet: "FORMAT parquet, COMPRESSION zstd",
|
|
38
|
+
csv: "FORMAT csv, HEADER true",
|
|
39
|
+
json: "FORMAT json, ARRAY true",
|
|
40
|
+
ndjson: "FORMAT json"
|
|
41
|
+
};
|
|
42
|
+
const SQLITE_TYPES = {
|
|
43
|
+
VARCHAR: "TEXT",
|
|
44
|
+
DATE: "TEXT",
|
|
45
|
+
INTEGER: "INTEGER",
|
|
46
|
+
BIGINT: "INTEGER",
|
|
47
|
+
DOUBLE: "REAL"
|
|
48
|
+
};
|
|
49
|
+
async function openSession() {
|
|
50
|
+
const instance = await DuckDBInstance.create(":memory:");
|
|
51
|
+
const connection = await instance.connect();
|
|
52
|
+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "gscdump-dump-"));
|
|
53
|
+
return {
|
|
54
|
+
connection,
|
|
55
|
+
tempDir,
|
|
56
|
+
async close() {
|
|
57
|
+
connection.closeSync();
|
|
58
|
+
instance.closeSync();
|
|
59
|
+
await fs.rm(tempDir, {
|
|
60
|
+
recursive: true,
|
|
61
|
+
force: true
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
async function countOf(connection, sql) {
|
|
67
|
+
const reader = await connection.runAndReadAll(sql);
|
|
68
|
+
return Number(reader.getRowsJS()[0]?.[0] ?? 0);
|
|
69
|
+
}
|
|
70
|
+
async function presentColumns(connection, source) {
|
|
71
|
+
const reader = await connection.runAndReadAll(`DESCRIBE SELECT * FROM ${readParquetSql(source.files)}`);
|
|
72
|
+
return new Set(reader.getRowObjectsJS().map((row) => String(row.column_name)));
|
|
73
|
+
}
|
|
74
|
+
async function entitySelect(session, site, dataset) {
|
|
75
|
+
const file = path.join(session.tempDir, `${siteDirName(site)}-${dataset.dataset}.parquet`);
|
|
76
|
+
await fs.writeFile(file, encodeRowsToParquetFlex(dataset.rows, { columns: dataset.columns }));
|
|
77
|
+
const columns = dataset.columns.map((column) => column.name).join(", ");
|
|
78
|
+
return `SELECT ${sqlString(site)} AS site, ${columns} FROM read_parquet(${sqlString(file)})`;
|
|
79
|
+
}
|
|
80
|
+
function entityColumns(dataset) {
|
|
81
|
+
return [{
|
|
82
|
+
name: "site",
|
|
83
|
+
type: "VARCHAR",
|
|
84
|
+
nullable: false
|
|
85
|
+
}, ...dataset.columns];
|
|
86
|
+
}
|
|
87
|
+
async function openDumpSink(outDir, format) {
|
|
88
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
89
|
+
if (format === "sqlite") return openSqliteSink(outDir);
|
|
90
|
+
if (format === "duckdb") return openDuckDBSink(outDir);
|
|
91
|
+
return openFileSink(outDir, format);
|
|
92
|
+
}
|
|
93
|
+
async function openFileSink(outDir, format) {
|
|
94
|
+
const session = await openSession();
|
|
95
|
+
const written = [];
|
|
96
|
+
const siteDir = createSiteDirNamer();
|
|
97
|
+
const copy = async (select, target) => {
|
|
98
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
99
|
+
const rows = await countOf(session.connection, `COPY (${select}) TO ${sqlString(target)} (${COPY_OPTIONS[format]})`);
|
|
100
|
+
written.push(target);
|
|
101
|
+
return rows;
|
|
102
|
+
};
|
|
103
|
+
const position = format !== "parquet";
|
|
104
|
+
return {
|
|
105
|
+
async writeTable(source) {
|
|
106
|
+
const select = sourceSelectSql(source, await presentColumns(session.connection, source), {
|
|
107
|
+
position,
|
|
108
|
+
dates: "date"
|
|
109
|
+
});
|
|
110
|
+
const target = path.join(outDir, siteDir(source.site), source.searchType, `${source.table}.${format}`);
|
|
111
|
+
return {
|
|
112
|
+
dataset: source.table,
|
|
113
|
+
searchType: source.searchType,
|
|
114
|
+
path: target,
|
|
115
|
+
rows: await copy(select, target)
|
|
116
|
+
};
|
|
117
|
+
},
|
|
118
|
+
async writeEntity(site, dataset) {
|
|
119
|
+
const target = path.join(outDir, siteDir(site), `${dataset.dataset}.${format}`);
|
|
120
|
+
return {
|
|
121
|
+
dataset: dataset.dataset,
|
|
122
|
+
path: target,
|
|
123
|
+
rows: await copy(await entitySelect(session, site, dataset), target)
|
|
124
|
+
};
|
|
125
|
+
},
|
|
126
|
+
async close() {
|
|
127
|
+
await session.close();
|
|
128
|
+
return Promise.all(written.map(async (file) => ({
|
|
129
|
+
path: file,
|
|
130
|
+
bytes: (await fs.stat(file)).size
|
|
131
|
+
})));
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
async function openDuckDBSink(outDir) {
|
|
136
|
+
const target = databasePath(outDir, "duckdb");
|
|
137
|
+
await fs.rm(target, { force: true });
|
|
138
|
+
await fs.rm(`${target}.wal`, { force: true });
|
|
139
|
+
const session = await openSession();
|
|
140
|
+
await session.connection.run(`ATTACH ${sqlString(target)} AS out`);
|
|
141
|
+
const created = /* @__PURE__ */ new Set();
|
|
142
|
+
const insert = async (name, columns, select) => {
|
|
143
|
+
if (!created.has(name)) {
|
|
144
|
+
const ddl = columns.map((column) => `${column.name} ${column.type}${column.nullable ? "" : " NOT NULL"}`).join(", ");
|
|
145
|
+
await session.connection.run(`CREATE TABLE out.${name} (${ddl})`);
|
|
146
|
+
created.add(name);
|
|
147
|
+
}
|
|
148
|
+
return countOf(session.connection, `INSERT INTO out.${name} BY NAME ${select}`);
|
|
149
|
+
};
|
|
150
|
+
return {
|
|
151
|
+
async writeTable(source) {
|
|
152
|
+
const select = sourceSelectSql(source, await presentColumns(session.connection, source), {
|
|
153
|
+
position: false,
|
|
154
|
+
dates: "date"
|
|
155
|
+
});
|
|
156
|
+
return {
|
|
157
|
+
dataset: source.table,
|
|
158
|
+
searchType: source.searchType,
|
|
159
|
+
path: target,
|
|
160
|
+
rows: await insert(source.table, exportColumns(source.table), select)
|
|
161
|
+
};
|
|
162
|
+
},
|
|
163
|
+
async writeEntity(site, dataset) {
|
|
164
|
+
return {
|
|
165
|
+
dataset: dataset.dataset,
|
|
166
|
+
path: target,
|
|
167
|
+
rows: await insert(dataset.dataset, entityColumns(dataset), await entitySelect(session, site, dataset))
|
|
168
|
+
};
|
|
169
|
+
},
|
|
170
|
+
async close() {
|
|
171
|
+
await session.connection.run("DETACH out");
|
|
172
|
+
await session.close();
|
|
173
|
+
return [{
|
|
174
|
+
path: target,
|
|
175
|
+
bytes: (await fs.stat(target)).size
|
|
176
|
+
}];
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function sqliteValue(value) {
|
|
181
|
+
if (value === null || value === void 0) return null;
|
|
182
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
183
|
+
if (value instanceof Date) return value.toISOString();
|
|
184
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") return value;
|
|
185
|
+
return JSON.stringify(value);
|
|
186
|
+
}
|
|
187
|
+
async function openSqliteSink(outDir) {
|
|
188
|
+
const target = databasePath(outDir, "sqlite");
|
|
189
|
+
await fs.rm(target, { force: true });
|
|
190
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
191
|
+
const db = new DatabaseSync(target);
|
|
192
|
+
const session = await openSession();
|
|
193
|
+
const created = /* @__PURE__ */ new Set();
|
|
194
|
+
const insert = async (name, columns, select) => {
|
|
195
|
+
if (!created.has(name)) {
|
|
196
|
+
const ddl = columns.map((column) => `"${column.name}" ${SQLITE_TYPES[column.type]}${column.nullable ? "" : " NOT NULL"}`).join(", ");
|
|
197
|
+
db.exec(`CREATE TABLE "${name}" (${ddl})`);
|
|
198
|
+
created.add(name);
|
|
199
|
+
}
|
|
200
|
+
const statement = db.prepare(`INSERT INTO "${name}" (${columns.map((column) => `"${column.name}"`).join(", ")}) VALUES (${columns.map(() => "?").join(", ")})`);
|
|
201
|
+
const result = await session.connection.stream(`SELECT ${columns.map((column) => column.name).join(", ")} FROM (${select})`);
|
|
202
|
+
let rows = 0;
|
|
203
|
+
db.exec("BEGIN");
|
|
204
|
+
try {
|
|
205
|
+
for await (const chunk of result.yieldRowsJs()) for (const row of chunk) {
|
|
206
|
+
statement.run(...row.map(sqliteValue));
|
|
207
|
+
rows++;
|
|
208
|
+
}
|
|
209
|
+
db.exec("COMMIT");
|
|
210
|
+
} catch (error) {
|
|
211
|
+
db.exec("ROLLBACK");
|
|
212
|
+
throw error;
|
|
213
|
+
}
|
|
214
|
+
return rows;
|
|
215
|
+
};
|
|
216
|
+
return {
|
|
217
|
+
async writeTable(source) {
|
|
218
|
+
const select = sourceSelectSql(source, await presentColumns(session.connection, source), {
|
|
219
|
+
position: false,
|
|
220
|
+
dates: "iso"
|
|
221
|
+
});
|
|
222
|
+
return {
|
|
223
|
+
dataset: source.table,
|
|
224
|
+
searchType: source.searchType,
|
|
225
|
+
path: target,
|
|
226
|
+
rows: await insert(source.table, exportColumns(source.table), select)
|
|
227
|
+
};
|
|
228
|
+
},
|
|
229
|
+
async writeEntity(site, dataset) {
|
|
230
|
+
return {
|
|
231
|
+
dataset: dataset.dataset,
|
|
232
|
+
path: target,
|
|
233
|
+
rows: await insert(dataset.dataset, entityColumns(dataset), await entitySelect(session, site, dataset))
|
|
234
|
+
};
|
|
235
|
+
},
|
|
236
|
+
async close() {
|
|
237
|
+
db.close();
|
|
238
|
+
await session.close();
|
|
239
|
+
return [{
|
|
240
|
+
path: target,
|
|
241
|
+
bytes: (await fs.stat(target)).size
|
|
242
|
+
}];
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
export { DUMP_FORMATS, createSiteDirNamer, databasePath, isDumpFormat, openDumpSink, siteDirName };
|
package/dist/error-handler.mjs
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import { isQueryError } from "gscdump/query";
|
|
1
|
+
import { isUsageError } from "./command-registry.mjs";
|
|
2
|
+
import { quotaStopOf } from "./quota-ledger.mjs";
|
|
4
3
|
import { isAnalysisError } from "@gscdump/analysis/errors";
|
|
5
4
|
import { isEngineError } from "@gscdump/engine/errors";
|
|
6
5
|
import { classifyError } from "gscdump/errors";
|
|
6
|
+
import { isQueryError } from "gscdump/query";
|
|
7
|
+
const HOSTED_KEY_REJECTED_REASON = "gscdump.com rejected the API key";
|
|
8
|
+
const HOSTED_KEY_REJECTED = `${HOSTED_KEY_REJECTED_REASON}. Run \`gscdump auth login --mode cloud --api-key KEY\` with a valid key.`;
|
|
7
9
|
const QUOTA_MESSAGE_RE = /quota|rate\s*limit/i;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const lines = [`\x1B[31m${error.message}\x1B[0m`];
|
|
11
|
-
let suggestion = "";
|
|
10
|
+
const ANSI_RE = /\x1B\[[0-9;]*m/g;
|
|
11
|
+
function suggestionForGscError(error) {
|
|
12
12
|
switch (error.kind) {
|
|
13
|
-
case "auth-expired":
|
|
14
|
-
|
|
15
|
-
break;
|
|
13
|
+
case "auth-expired": return "Run `gscdump auth login` to sign in again.";
|
|
14
|
+
case "permission-denied": return "Check that this Google account is a full user or owner of the Site. Run `gscdump sites` to see your Sites.";
|
|
16
15
|
case "rate-limited": {
|
|
17
16
|
const retryIn = error.retryAfter ? `${error.retryAfter}s` : "a few minutes";
|
|
18
|
-
if (QUOTA_MESSAGE_RE.test(error.message))
|
|
19
|
-
|
|
20
|
-
break;
|
|
17
|
+
if (!QUOTA_MESSAGE_RE.test(error.message)) return `Google rate limited the request. Try again in ${retryIn}.`;
|
|
18
|
+
return error.message.includes("Indexing API") ? "The Indexing API quota is used up (about 200 per day). Try again tomorrow." : `A Google quota or rate limit was reached. Try again in ${retryIn}.`;
|
|
21
19
|
}
|
|
20
|
+
case "not-found":
|
|
21
|
+
case "validation":
|
|
22
|
+
case "storage":
|
|
23
|
+
case "transport": return "";
|
|
22
24
|
}
|
|
23
|
-
if (suggestion) lines.push("", suggestion);
|
|
24
|
-
return lines.join("\n");
|
|
25
25
|
}
|
|
26
26
|
function extractQueryError(error) {
|
|
27
27
|
if (isQueryError(error)) return error;
|
|
28
|
-
const tagged = error
|
|
28
|
+
const tagged = error?.queryError;
|
|
29
29
|
return isQueryError(tagged) ? tagged : null;
|
|
30
30
|
}
|
|
31
31
|
function extractEngineError(error) {
|
|
32
32
|
if (isEngineError(error)) return error;
|
|
33
|
-
const tagged = error
|
|
33
|
+
const tagged = error?.engineError;
|
|
34
34
|
return isEngineError(tagged) ? tagged : null;
|
|
35
35
|
}
|
|
36
36
|
function extractAnalysisError(error) {
|
|
37
37
|
if (isAnalysisError(error)) return error;
|
|
38
|
-
const tagged = error
|
|
38
|
+
const tagged = error?.analysisError;
|
|
39
39
|
return isAnalysisError(tagged) ? tagged : null;
|
|
40
40
|
}
|
|
41
41
|
function suggestionForTypedError(error) {
|
|
@@ -52,7 +52,7 @@ function suggestionForTypedError(error) {
|
|
|
52
52
|
}
|
|
53
53
|
const analysis = extractAnalysisError(error);
|
|
54
54
|
if (analysis) switch (analysis.kind) {
|
|
55
|
-
case "missing-report-param": return `Report "${analysis.report}" needs
|
|
55
|
+
case "missing-report-param": return `Report "${analysis.report}" needs ${flagForParam(analysis.param)}.`;
|
|
56
56
|
case "missing-comparison-window": return `Report "${analysis.report}" is a comparison report; pass --vs prev-period (or --vs yoy).`;
|
|
57
57
|
case "missing-brand-terms": return "Pass --brand-terms to segment branded vs non-branded queries.";
|
|
58
58
|
case "unknown-report": return `Unknown report. Available: ${analysis.available.join(", ")}.`;
|
|
@@ -81,25 +81,95 @@ var LocalStoreUnsupportedError = class extends Error {
|
|
|
81
81
|
this.mode = mode;
|
|
82
82
|
}
|
|
83
83
|
};
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
if (
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
84
|
+
function isAuthError(err) {
|
|
85
|
+
const msg = (err instanceof Error ? err.message : String(err ?? "")).toLowerCase();
|
|
86
|
+
if (!msg) return false;
|
|
87
|
+
return /\b(?:401|unauthorized|invalid_grant|invalid_token|insufficient.*scope|invalid_client|token has been expired|token has been revoked)\b/.test(msg) || msg.includes("oauth2.googleapis.com/token");
|
|
88
|
+
}
|
|
89
|
+
function flagForParam(param) {
|
|
90
|
+
return {
|
|
91
|
+
prevStartDate: "--prev-start",
|
|
92
|
+
prevEndDate: "--prev-end",
|
|
93
|
+
startDate: "--start",
|
|
94
|
+
endDate: "--end"
|
|
95
|
+
}[param] ?? `--${param.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)}`;
|
|
96
|
+
}
|
|
97
|
+
const PARAM_NAME_RE = /\b(prevStartDate|prevEndDate|brandTerms)\b/g;
|
|
98
|
+
const DEFECT_ERRORS = /* @__PURE__ */ new Set([
|
|
99
|
+
"TypeError",
|
|
100
|
+
"ReferenceError",
|
|
101
|
+
"RangeError",
|
|
102
|
+
"SyntaxError",
|
|
103
|
+
"EvalError",
|
|
104
|
+
"URIError"
|
|
105
|
+
]);
|
|
106
|
+
function describeCliError(error) {
|
|
107
|
+
if (isUsageError(error)) return {
|
|
108
|
+
kind: "usage",
|
|
109
|
+
message: error.message
|
|
110
|
+
};
|
|
111
|
+
if (error instanceof Error && error.routeStop !== void 0) return {
|
|
112
|
+
kind: "stop",
|
|
113
|
+
message: error.message
|
|
114
|
+
};
|
|
115
|
+
if (error instanceof Error && quotaStopOf(error)) return {
|
|
116
|
+
kind: "stop",
|
|
117
|
+
message: error.message
|
|
118
|
+
};
|
|
119
|
+
if (error instanceof TypeError && error.message === "fetch failed") {
|
|
120
|
+
const cause = error.cause;
|
|
121
|
+
return {
|
|
122
|
+
kind: "expected",
|
|
123
|
+
message: `Network request failed: ${cause instanceof Error ? cause.message : "no response"}`,
|
|
124
|
+
hint: "Check your connection, then try again.",
|
|
125
|
+
showAuthSources: false
|
|
126
|
+
};
|
|
97
127
|
}
|
|
98
|
-
if (
|
|
99
|
-
|
|
100
|
-
|
|
128
|
+
if (error instanceof Error && DEFECT_ERRORS.has(error.constructor.name)) return {
|
|
129
|
+
kind: "defect",
|
|
130
|
+
message: `${error.name}: ${error.message}`,
|
|
131
|
+
stack: error.stack ?? ""
|
|
132
|
+
};
|
|
133
|
+
if (error instanceof LocalStoreUnsupportedError) return {
|
|
134
|
+
kind: "expected",
|
|
135
|
+
message: error.message,
|
|
136
|
+
hint: error.mode === "local" ? "Pass --live to run against the GSC API." : "",
|
|
137
|
+
showAuthSources: false
|
|
138
|
+
};
|
|
139
|
+
const classified = classifyError(error);
|
|
140
|
+
if (classified.message === HOSTED_KEY_REJECTED) return {
|
|
141
|
+
kind: "expected",
|
|
142
|
+
message: classified.message,
|
|
143
|
+
hint: "",
|
|
144
|
+
showAuthSources: false
|
|
145
|
+
};
|
|
146
|
+
return {
|
|
147
|
+
kind: "expected",
|
|
148
|
+
message: classified.message.replace(PARAM_NAME_RE, (name) => flagForParam(name)),
|
|
149
|
+
hint: suggestionForTypedError(error) || suggestionForGscError(classified),
|
|
150
|
+
showAuthSources: classified.kind === "auth-expired" || classified.kind === "permission-denied" || isAuthError(error)
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
async function reportCliError(error, options) {
|
|
154
|
+
const report = describeCliError(error);
|
|
155
|
+
const red = (text) => options.color ? `\x1B[31m${text}\x1B[0m` : text;
|
|
156
|
+
const lines = [];
|
|
157
|
+
switch (report.kind) {
|
|
158
|
+
case "usage":
|
|
159
|
+
if (options.usage) lines.push(await options.usage(), "");
|
|
160
|
+
lines.push(red(report.message));
|
|
161
|
+
break;
|
|
162
|
+
case "stop":
|
|
163
|
+
lines.push(report.message);
|
|
164
|
+
break;
|
|
165
|
+
case "expected":
|
|
166
|
+
lines.push(red(`Error: ${report.message}`));
|
|
167
|
+
if (report.hint) lines.push(report.hint);
|
|
168
|
+
if (report.showAuthSources && options.authSources) lines.push("", await options.authSources());
|
|
169
|
+
break;
|
|
170
|
+
case "defect": lines.push(red(`Unexpected error. Report it at https://github.com/harlan-zw/gscdump/issues`), report.stack || report.message);
|
|
101
171
|
}
|
|
102
|
-
|
|
103
|
-
|
|
172
|
+
const text = lines.join("\n");
|
|
173
|
+
(options.write ?? ((value) => console.error(value)))(options.color ? text : text.replace(ANSI_RE, ""));
|
|
104
174
|
}
|
|
105
|
-
export { LocalStoreUnsupportedError,
|
|
175
|
+
export { HOSTED_KEY_REJECTED, HOSTED_KEY_REJECTED_REASON, LocalStoreUnsupportedError, describeCliError, flagForParam, isAuthError, reportCliError };
|
package/dist/filters.mjs
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { and } from "gscdump/query";
|
|
2
|
+
import { toPath } from "@gscdump/engine/ingest";
|
|
3
|
+
const FILTER_DIMS = [
|
|
4
|
+
"query",
|
|
5
|
+
"page",
|
|
6
|
+
"country",
|
|
7
|
+
"device",
|
|
8
|
+
"searchAppearance"
|
|
9
|
+
];
|
|
10
|
+
function splitOperator(raw) {
|
|
11
|
+
if (raw.startsWith("!~")) return {
|
|
12
|
+
operator: "notContains",
|
|
13
|
+
value: raw.slice(2)
|
|
14
|
+
};
|
|
15
|
+
if (raw.startsWith("!re:")) return {
|
|
16
|
+
operator: "excludingRegex",
|
|
17
|
+
value: raw.slice(4)
|
|
18
|
+
};
|
|
19
|
+
if (raw.startsWith("!")) return {
|
|
20
|
+
operator: "notEquals",
|
|
21
|
+
value: raw.slice(1)
|
|
22
|
+
};
|
|
23
|
+
if (raw.startsWith("~")) return {
|
|
24
|
+
operator: "contains",
|
|
25
|
+
value: raw.slice(1)
|
|
26
|
+
};
|
|
27
|
+
if (raw.startsWith("re:")) return {
|
|
28
|
+
operator: "includingRegex",
|
|
29
|
+
value: raw.slice(3)
|
|
30
|
+
};
|
|
31
|
+
if (raw.startsWith("contains:")) return {
|
|
32
|
+
operator: "contains",
|
|
33
|
+
value: raw.slice(9)
|
|
34
|
+
};
|
|
35
|
+
if (raw.startsWith("eq:")) return {
|
|
36
|
+
operator: "equals",
|
|
37
|
+
value: raw.slice(3)
|
|
38
|
+
};
|
|
39
|
+
return {
|
|
40
|
+
operator: "equals",
|
|
41
|
+
value: raw
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function parsePageRef(value) {
|
|
45
|
+
if (!/^https?:\/\//i.test(value)) return { path: value };
|
|
46
|
+
const origin = URL.canParse(value) ? new URL(value).origin : void 0;
|
|
47
|
+
return origin ? {
|
|
48
|
+
path: toPath(value),
|
|
49
|
+
origin
|
|
50
|
+
} : { path: value };
|
|
51
|
+
}
|
|
52
|
+
function parseFilterArg(dim, raw) {
|
|
53
|
+
const { operator, value } = splitOperator(raw);
|
|
54
|
+
if (operator === "includingRegex" || operator === "excludingRegex") return {
|
|
55
|
+
kind: "regex",
|
|
56
|
+
dim,
|
|
57
|
+
operator,
|
|
58
|
+
pattern: value
|
|
59
|
+
};
|
|
60
|
+
if (dim === "page") return {
|
|
61
|
+
kind: "page",
|
|
62
|
+
operator,
|
|
63
|
+
page: parsePageRef(value)
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
kind: "match",
|
|
67
|
+
dim,
|
|
68
|
+
operator,
|
|
69
|
+
value
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function parseFilterArgs(args) {
|
|
73
|
+
const argName = {
|
|
74
|
+
query: "query",
|
|
75
|
+
page: "page",
|
|
76
|
+
country: "country",
|
|
77
|
+
device: "device",
|
|
78
|
+
searchAppearance: "search-appearance"
|
|
79
|
+
};
|
|
80
|
+
return FILTER_DIMS.flatMap((dim) => {
|
|
81
|
+
const raw = args[argName[dim]] ?? args[dim];
|
|
82
|
+
return raw == null || raw === "" ? [] : [parseFilterArg(dim, String(raw))];
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function filterDimensions(filters) {
|
|
86
|
+
return [...new Set(filters.map((filter) => filter.kind === "page" ? "page" : filter.dim))];
|
|
87
|
+
}
|
|
88
|
+
function leaf(dimension, operator, expression) {
|
|
89
|
+
return {
|
|
90
|
+
_constraints: {},
|
|
91
|
+
_filters: [{
|
|
92
|
+
dimension,
|
|
93
|
+
operator,
|
|
94
|
+
expression
|
|
95
|
+
}]
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function combine(leaves) {
|
|
99
|
+
if (leaves.length <= 1) return leaves[0];
|
|
100
|
+
return and(...leaves);
|
|
101
|
+
}
|
|
102
|
+
function localLeaf(filter) {
|
|
103
|
+
if (filter.kind === "page") return leaf("page", filter.operator, filter.page.path);
|
|
104
|
+
return filter.kind === "match" ? leaf(filter.dim, filter.operator, filter.value) : leaf(filter.dim, filter.operator, filter.pattern);
|
|
105
|
+
}
|
|
106
|
+
function toLocalFilter(filters) {
|
|
107
|
+
return combine(filters.map(localLeaf));
|
|
108
|
+
}
|
|
109
|
+
function escapeRegex(value) {
|
|
110
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
111
|
+
}
|
|
112
|
+
function livePageLeaf(operator, page, siteUrl) {
|
|
113
|
+
if (page.origin) return leaf("page", operator, `${page.origin}${page.path}`);
|
|
114
|
+
if (operator === "contains" || operator === "notContains") return leaf("page", operator, page.path);
|
|
115
|
+
const path = page.path.startsWith("/") ? page.path : `/${page.path}`;
|
|
116
|
+
if (siteUrl.startsWith("sc-domain:")) return leaf("page", operator === "equals" ? "includingRegex" : "excludingRegex", `^https?://[^/]+${escapeRegex(path)}$`);
|
|
117
|
+
const url = new URL(siteUrl);
|
|
118
|
+
const propertyPath = url.pathname.replace(/\/+$/, "");
|
|
119
|
+
if (propertyPath && (path === propertyPath || path.startsWith(`${propertyPath}/`))) return leaf("page", operator, `${url.origin}${path}`);
|
|
120
|
+
return leaf("page", operator, `${siteUrl.endsWith("/") ? siteUrl.slice(0, -1) : siteUrl}${path}`);
|
|
121
|
+
}
|
|
122
|
+
function toLiveFilter(filters, siteUrl) {
|
|
123
|
+
return combine(filters.map((filter) => filter.kind === "page" ? livePageLeaf(filter.operator, filter.page, siteUrl) : localLeaf(filter)));
|
|
124
|
+
}
|
|
125
|
+
export { FILTER_DIMS, filterDimensions, parseFilterArg, parseFilterArgs, parsePageRef, toLiveFilter, toLocalFilter };
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { getCloudAccount, parseAuthentication, resolveAuthentication } from "./auth-state.mjs";
|
|
2
|
+
import { loadConfig } from "./config.mjs";
|
|
3
|
+
import { resolveCliEnvironment } from "./environment.mjs";
|
|
4
|
+
import { logger } from "./utils.mjs";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { createGscdumpV1Client } from "@gscdump/sdk/v1";
|
|
7
|
+
const HOSTED_ARGS = {
|
|
8
|
+
"site": {
|
|
9
|
+
type: "string",
|
|
10
|
+
alias: "s",
|
|
11
|
+
description: "Site URL (e.g., example.com, sc-domain:example.com, or https://example.com/)"
|
|
12
|
+
},
|
|
13
|
+
"api-root": {
|
|
14
|
+
type: "string",
|
|
15
|
+
description: "Hosted API root; defaults to saved cloud authentication or https://gscdump.com/api"
|
|
16
|
+
},
|
|
17
|
+
"api-key": {
|
|
18
|
+
type: "string",
|
|
19
|
+
description: "Hosted API key; defaults to GSCDUMP_API_KEY"
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
function siteKey(value) {
|
|
23
|
+
const text = value.trim();
|
|
24
|
+
if (text.startsWith("sc-domain:")) return text.slice(10).toLowerCase().replace(/\.$/, "") || void 0;
|
|
25
|
+
const url = URL.parse(/^[a-z][a-z\d+.-]*:\/\//i.test(text) ? text : `https://${text}`);
|
|
26
|
+
if (!url || !url.hostname) return void 0;
|
|
27
|
+
return `${url.hostname.toLowerCase()}${url.pathname.replace(/\/+$/, "")}`;
|
|
28
|
+
}
|
|
29
|
+
function matchHostedSite(sites, target) {
|
|
30
|
+
if (!target) return sites.length === 1 ? {
|
|
31
|
+
kind: "found",
|
|
32
|
+
site: sites[0]
|
|
33
|
+
} : {
|
|
34
|
+
kind: "missing",
|
|
35
|
+
target,
|
|
36
|
+
sites: [...sites]
|
|
37
|
+
};
|
|
38
|
+
const exact = sites.find((site) => site.siteId === target || site.siteUrl === target);
|
|
39
|
+
if (exact) return {
|
|
40
|
+
kind: "found",
|
|
41
|
+
site: exact
|
|
42
|
+
};
|
|
43
|
+
const key = siteKey(target);
|
|
44
|
+
const matches = key === void 0 ? [] : sites.filter((site) => siteKey(site.siteUrl) === key);
|
|
45
|
+
if (matches.length === 1) return {
|
|
46
|
+
kind: "found",
|
|
47
|
+
site: matches[0]
|
|
48
|
+
};
|
|
49
|
+
if (matches.length > 1) return {
|
|
50
|
+
kind: "ambiguous",
|
|
51
|
+
target,
|
|
52
|
+
matches
|
|
53
|
+
};
|
|
54
|
+
return {
|
|
55
|
+
kind: "missing",
|
|
56
|
+
target,
|
|
57
|
+
sites: [...sites]
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function describeMatchFailure(match) {
|
|
61
|
+
if (match.kind === "ambiguous") return `Multiple hosted Sites match "${match.target}": ${match.matches.map((site) => site.siteUrl).join(", ")}. Pass the exact Site URL with --site.`;
|
|
62
|
+
if (match.sites.length === 0) return "Your hosted account has no Sites. Add a Site at https://gscdump.com.";
|
|
63
|
+
const available = `Hosted Sites: ${match.sites.map((site) => site.siteUrl).join(", ")}.`;
|
|
64
|
+
return match.target ? `No hosted Site matches "${match.target}". ${available}` : `Pass --site. ${available}`;
|
|
65
|
+
}
|
|
66
|
+
function fail(message) {
|
|
67
|
+
logger.error(message);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
|
70
|
+
async function resolveHostedSite(args, command) {
|
|
71
|
+
if (args._?.length) fail(`\`gscdump ${command.name}\` does not accept a positional Site ID. Pass --site instead.`);
|
|
72
|
+
const environment = resolveCliEnvironment().values;
|
|
73
|
+
const authentication = args["api-key"] ? parseAuthentication({
|
|
74
|
+
_tag: "Cloud",
|
|
75
|
+
apiKey: args["api-key"],
|
|
76
|
+
apiRoot: String(args["api-root"] || environment.GSCDUMP_API_ROOT || "https://gscdump.com/api")
|
|
77
|
+
}) : await resolveAuthentication();
|
|
78
|
+
if (authentication._tag !== "Cloud") fail(`\`gscdump ${command.name}\` needs hosted authentication. Run \`gscdump auth login --mode cloud\` or pass --api-key. With local authentication, ${command.localAlternative}.`);
|
|
79
|
+
if (args["api-root"] && String(args["api-root"]).replace(/\/+$/, "") !== authentication.apiRoot) fail("The API root changed. Pass --api-key for the new API root.");
|
|
80
|
+
const target = args.site ? String(args.site) : (await loadConfig()).defaultSite;
|
|
81
|
+
const match = matchHostedSite((await getCloudAccount(authentication)).sites, target);
|
|
82
|
+
if (match.kind !== "found") fail(describeMatchFailure(match));
|
|
83
|
+
return {
|
|
84
|
+
client: createGscdumpV1Client({
|
|
85
|
+
apiRoot: authentication.apiRoot,
|
|
86
|
+
credential: authentication.apiKey
|
|
87
|
+
}),
|
|
88
|
+
site: match.site
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
export { HOSTED_ARGS, matchHostedSite, resolveHostedSite };
|