@gscdump/cli 3.7.0 → 4.0.0

package/dist/sync-run.mjs

@@ -0,0 +1,96 @@
+import process from "node:process";
+import fs from "node:fs/promises";
+import path from "node:path";
+const SYNC_HEARTBEAT_STALE_MS = 12e4;
+const HEARTBEAT_INTERVAL_MS = 1e4;
+function syncRunStatus(record, probe) {
+	if (!record) return { kind: "none" };
+	if (record.finishedAt !== void 0) return {
+		kind: "finished",
+		record
+	};
+	if (!probe.isAlive(record.pid)) return {
+		kind: "stale",
+		record,
+		reason: "process-gone"
+	};
+	if (probe.now - record.heartbeatAt > 12e4) return {
+		kind: "stale",
+		record,
+		reason: "no-heartbeat"
+	};
+	return {
+		kind: "running",
+		record
+	};
+}
+function isProcessAlive(pid) {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (error) {
+		return error.code === "EPERM";
+	}
+}
+function syncRunPath(dataDir) {
+	return path.join(dataDir, "sync-run.json");
+}
+async function readSyncRun(dataDir) {
+	const body = await fs.readFile(syncRunPath(dataDir), "utf8").catch((error) => {
+		if (error.code === "ENOENT") return void 0;
+		throw error;
+	});
+	return body === void 0 ? void 0 : JSON.parse(body);
+}
+async function writeSyncRun(dataDir, record) {
+	await fs.mkdir(dataDir, { recursive: true });
+	const file = syncRunPath(dataDir);
+	const temporary = `${file}.${record.pid}.tmp`;
+	await fs.writeFile(temporary, JSON.stringify(record, null, 2), "utf8");
+	await fs.rename(temporary, file);
+}
+async function startSyncRun(opts) {
+	const now = opts.now ?? Date.now;
+	const record = {
+		pid: process.pid,
+		startedAt: now(),
+		heartbeatAt: now(),
+		sites: opts.sites,
+		planned: 0,
+		done: 0
+	};
+	let writing = Promise.resolve();
+	const save = () => {
+		record.heartbeatAt = now();
+		const snapshot = { ...record };
+		writing = writing.then(() => writeSyncRun(opts.dataDir, snapshot));
+		return writing;
+	};
+	await save();
+	const timer = setInterval(() => {
+		save().then(() => opts.onHeartbeat?.()).catch((error) => {
+			process.stderr.write(`gscdump: could not update the sync-run record: ${error.message}\n`);
+		});
+	}, HEARTBEAT_INTERVAL_MS);
+	timer.unref();
+	return {
+		record,
+		setSite(site) {
+			record.site = site;
+		},
+		addPlanned(n) {
+			record.planned += n;
+		},
+		tick(n = 1) {
+			record.done += n;
+		},
+		save,
+		async finish(outcome) {
+			clearInterval(timer);
+			record.finishedAt = now();
+			record.outcome = outcome;
+			await save();
+		}
+	};
+}
+export { SYNC_HEARTBEAT_STALE_MS, isProcessAlive, readSyncRun, startSyncRun, syncRunPath, syncRunStatus };

package/dist/table-sources.mjs

@@ -0,0 +1,57 @@
+import { siteUrlForId } from "./store-sites.mjs";
+import path from "node:path";
+import { SCHEMAS } from "@gscdump/engine/schema";
+function groupTableSources(entries, dataDir, siteMap) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const entry of entries) {
+		if (!entry.siteId || entry.rowCount === 0) continue;
+		const searchType = entry.searchType ?? "web";
+		const key = `${entry.table}\u0000${entry.siteId}\u0000${searchType}`;
+		let group = groups.get(key);
+		if (!group) {
+			group = {
+				table: entry.table,
+				siteId: entry.siteId,
+				site: siteUrlForId(siteMap, entry.siteId),
+				searchType,
+				files: [],
+				rows: 0
+			};
+			groups.set(key, group);
+		}
+		group.files.push(path.join(dataDir, entry.objectKey));
+		group.rows += entry.rowCount;
+	}
+	return [...groups.values()].sort((a, b) => a.table.localeCompare(b.table) || a.site.localeCompare(b.site) || a.searchType.localeCompare(b.searchType));
+}
+function exportColumns(table) {
+	return [
+		{
+			name: "site",
+			type: "VARCHAR",
+			nullable: false
+		},
+		{
+			name: "search_type",
+			type: "VARCHAR",
+			nullable: false
+		},
+		...SCHEMAS[table].columns
+	];
+}
+function sqlString(value) {
+	return `'${value.replace(/'/g, "''")}'`;
+}
+function sourceSelectSql(source, present, opts) {
+	const columns = SCHEMAS[source.table].columns.map((column) => {
+		if (!present.has(column.name)) return `NULL::${column.type} AS ${column.name}`;
+		if (column.type !== "DATE") return column.name;
+		return opts.dates === "date" ? `CAST(${column.name} AS DATE) AS ${column.name}` : `strftime(CAST(${column.name} AS DATE), '%Y-%m-%d') AS ${column.name}`;
+	});
+	const position = opts.position ? [`sum_position / NULLIF(impressions, 0) + 1 AS position`] : [];
+	return `SELECT ${sqlString(source.site)} AS site, ${sqlString(source.searchType)} AS search_type, ${[...columns, ...position].join(", ")} FROM ${readParquetSql(source.files)}`;
+}
+function readParquetSql(files) {
+	return `read_parquet([${files.map(sqlString).join(", ")}], union_by_name = true)`;
+}
+export { exportColumns, groupTableSources, readParquetSql, sourceSelectSql, sqlString };

package/dist/token-info.mjs

@@ -0,0 +1,54 @@
+import { ofetch } from "ofetch";
+const TOKENINFO_URL = "https://oauth2.googleapis.com/tokeninfo";
+const TIMEOUT_MS = 5e3;
+const TOKEN_PATTERNS = [
+	/ya29\.[\w.-]+/g,
+	/1\/\/[\w.-]{20,}/g,
+	/(access_token|refresh_token)=[^&\s"']+/gi
+];
+function redactTokens(text, secrets = []) {
+	let out = text;
+	for (const secret of secrets) if (secret) out = out.split(secret).join("<redacted>");
+	for (const pattern of TOKEN_PATTERNS) out = out.replace(pattern, (match) => match.includes("=") ? `${match.slice(0, match.indexOf("="))}=<redacted>` : "<redacted>");
+	return out;
+}
+function describeFailure(error) {
+	const data = error?.data;
+	const reason = typeof data?.error_description === "string" ? data.error_description : typeof data?.error === "string" ? data.error : void 0;
+	if (reason) return reason;
+	return error instanceof Error ? error.message : String(error);
+}
+async function fetchTokenInfo(accessToken, request = (url, init) => ofetch(url, init)) {
+	return request(TOKENINFO_URL, {
+		method: "POST",
+		body: new URLSearchParams({ access_token: accessToken }),
+		timeout: TIMEOUT_MS
+	}).then((info) => ({
+		kind: "valid",
+		info
+	}), (error) => ({
+		kind: "invalid",
+		detail: redactTokens(describeFailure(error), [accessToken])
+	}));
+}
+async function currentAccessToken(auth) {
+	if (typeof auth === "string") return {
+		kind: "token",
+		token: auth
+	};
+	if (!("getAccessToken" in auth) || typeof auth.getAccessToken !== "function") return {
+		kind: "failed",
+		detail: "These credentials cannot produce an access token."
+	};
+	return auth.getAccessToken().then((result) => result.token ? {
+		kind: "token",
+		token: result.token
+	} : {
+		kind: "failed",
+		detail: "Google returned no access token."
+	}, (error) => ({
+		kind: "failed",
+		detail: redactTokens(describeFailure(error))
+	}));
+}
+export { currentAccessToken, fetchTokenInfo, redactTokens };

package/dist/utils.mjs

@@ -2,19 +2,12 @@ import { useCliRuntime } from "./runtime.mjs";
 import { version } from "./package.mjs";
 import process from "node:process";
 import fs from "node:fs/promises";
-import os from "node:os";
 import { SearchTypes } from "gscdump/query";
+import os from "node:os";
 import { Buffer } from "node:buffer";
+import { MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
 const ALL_SEARCH_TYPES = Object.values(SearchTypes);
 const VERSION = version;
-function noSubcommandSelected(parent, subNames) {
-	const argv = useCliRuntime().rawArgs;
-	const idx = argv.indexOf(parent);
-	if (idx < 0) return true;
-	const next = argv[idx + 1];
-	if (!next) return true;
-	return !subNames.includes(next);
-}
 const logger = new Proxy({}, {
 	get(_target, property) {
 		const active = useCliRuntime().logger;
@@ -65,6 +58,13 @@ function parseSearchType(value, flag = "--search-type") {
 	}
 	return v;
 }
+function parseNameList(value, known, flag) {
+	const names = String(value ?? "").split(",").map((name) => name.trim()).filter(Boolean);
+	const unknown = names.filter((name) => !known.includes(name));
+	if (unknown.length > 0) throw new Error(`Unknown ${flag} name: ${unknown.join(", ")}. Valid names: ${known.join(", ")}.`);
+	if (names.length === 0) throw new Error(`${flag} needs at least one name. Valid names: ${known.join(", ")}.`);
+	return names;
+}
 function parseIntegerOption(value, flag, minimum = 1) {
 	if (value === void 0 || value === null) return void 0;
 	const text = typeof value === "number" || typeof value === "string" ? String(value).trim() : "";
@@ -72,6 +72,11 @@ function parseIntegerOption(value, flag, minimum = 1) {
 	if (!/^\d+$/.test(text) || !Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`${flag} must be ${minimum === 0 ? "a non-negative integer" : "a positive integer"}.`);
 	return parsed;
 }
+function parseFetchBudget(value) {
+	const budget = parseIntegerOption(value, "--fetch-budget");
+	if (budget !== void 0 && budget > MAX_FETCH_BUDGET) throw new Error(`--fetch-budget must be at most ${MAX_FETCH_BUDGET}. Larger fetches use up the GSC quota.`);
+	return budget;
+}
 const ANSI_RE = /\x1B\[[0-9;]*m/g;
 function setNoColor(disable) {
 	if (disable) useCliRuntime().colorEnabled = false;
@@ -110,6 +115,9 @@ function color(c, s) {
 function dim(s) {
 	return color("dim", s);
 }
+function red(s) {
+	return color("red", s);
+}
 function cyan(s) {
 	return color("cyan", s);
 }
@@ -174,14 +182,15 @@ function toCSV(data, columns) {
 		return str.includes(",") || str.includes("\"") || str.includes("\n") ? `"${str.replace(/"/g, "\"\"")}"` : str;
 	}).join(","))].join("\n");
 }
-async function readUrlList(args) {
-	if (args.file) return (await fs.readFile(String(args.file), "utf-8")).split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
-	if (args.urls) return (Array.isArray(args.urls) ? args.urls : [args.urls]).map(String).filter(Boolean);
-	if (!process.stdin.isTTY) {
-		const chunks = [];
-		for await (const chunk of process.stdin) chunks.push(chunk);
-		return Buffer.concat(chunks).toString("utf-8").split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
-	}
-	return [];
+function parseUrlLines(text) {
+	return text.split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
+}
+async function readUrlList(input) {
+	const urls = (input.positionals ?? []).map(String).filter(Boolean);
+	if (input.file) urls.push(...parseUrlLines(await fs.readFile(String(input.file), "utf-8")));
+	if (urls.length > 0 || input.file || process.stdin.isTTY) return urls;
+	const chunks = [];
+	for await (const chunk of process.stdin) chunks.push(chunk);
+	return parseUrlLines(Buffer.concat(chunks).toString("utf-8"));
 }
-export { ALL_SEARCH_TYPES, OUTPUT_ARGS, VERSION, applyOutputMode, clearLine, color, cyan, dim, displayPath, formatAge, isColorEnabled, logger, noSubcommandSelected, parseIntegerOption, parseSearchType, progressBar, readUrlList, runWithConcurrency, setNoColor, setQuiet, showSplash, toCSV, withConfiguredOutput };
+export { ALL_SEARCH_TYPES, OUTPUT_ARGS, VERSION, applyOutputMode, clearLine, color, cyan, dim, displayPath, formatAge, isColorEnabled, logger, parseFetchBudget, parseIntegerOption, parseNameList, parseSearchType, progressBar, readUrlList, red, runWithConcurrency, setNoColor, setQuiet, showSplash, toCSV, withConfiguredOutput };

package/dist/window.mjs

@@ -0,0 +1,159 @@
+import { err, ok } from "gscdump/result";
+import { resolveWindow } from "@gscdump/engine/period";
+const PERIOD_ALIASES = {
+	"7d": "last-7d",
+	"28d": "last-28d",
+	"30d": "last-30d",
+	"90d": "last-90d",
+	"180d": "last-180d",
+	"365d": "last-365d",
+	"last-7d": "last-7d",
+	"last-28d": "last-28d",
+	"last-30d": "last-30d",
+	"last-90d": "last-90d",
+	"last-180d": "last-180d",
+	"last-365d": "last-365d",
+	"mtd": "mtd",
+	"qtd": "qtd",
+	"ytd": "ytd",
+	"last-quarter": "last-quarter",
+	"custom": "custom"
+};
+const PERIOD_FLAGS = [
+	"7d",
+	"28d",
+	"30d",
+	"90d",
+	"180d",
+	"365d",
+	"mtd",
+	"qtd",
+	"ytd",
+	"last-quarter",
+	"custom"
+];
+const COMPARISON_ALIASES = {
+	"none": "none",
+	"prev": "prev-period",
+	"prev-period": "prev-period",
+	"prior": "prev-period",
+	"prior-period": "prev-period",
+	"yoy": "yoy"
+};
+const COMPARISON_FLAGS = [
+	"none",
+	"prev-period",
+	"yoy"
+];
+const DEFAULT_WINDOW = {
+	preset: "last-28d",
+	comparison: "none"
+};
+const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
+function isCalendarDate(value) {
+	if (!ISO_DATE_RE.test(value)) return false;
+	const date = /* @__PURE__ */ new Date(`${value}T00:00:00Z`);
+	return Number.isFinite(date.getTime()) && date.toISOString().slice(0, 10) === value;
+}
+function checkDate(flag, value) {
+	if (value === void 0 || isCalendarDate(value)) return void 0;
+	return {
+		kind: "invalid-date",
+		flag,
+		value,
+		message: `Invalid ${flag} "${value}". Use a calendar date in YYYY-MM-DD format.`
+	};
+}
+function checkWindowFlags(flags) {
+	for (const [flag, value] of [
+		["--start", flags.start],
+		["--end", flags.end],
+		["--prev-start", flags.prevStart],
+		["--prev-end", flags.prevEnd]
+	]) {
+		const invalid = checkDate(flag, value);
+		if (invalid) return invalid;
+	}
+	const preset = flags.period ? PERIOD_ALIASES[flags.period.toLowerCase()] : void 0;
+	if (flags.period && !preset) return {
+		kind: "unknown-period",
+		value: flags.period,
+		message: `Unknown --period "${flags.period}". Supported: ${PERIOD_FLAGS.join(", ")}.`
+	};
+	if (flags.vs && !COMPARISON_ALIASES[flags.vs.toLowerCase()]) return {
+		kind: "unknown-comparison",
+		value: flags.vs,
+		message: `Unknown --vs "${flags.vs}". Supported: ${COMPARISON_FLAGS.join(", ")}.`
+	};
+	if (flags.prevStart === void 0 !== (flags.prevEnd === void 0)) return {
+		kind: "partial-comparison",
+		message: "Pass --prev-start and --prev-end together."
+	};
+	if (preset && preset !== "custom" && (flags.start !== void 0 || flags.end !== void 0)) return {
+		kind: "period-conflict",
+		message: `--period ${flags.period} sets its own dates. Remove --start/--end, or pass --period custom.`
+	};
+	if (flags.start !== void 0 && flags.end !== void 0 && flags.start > flags.end) return {
+		kind: "inverted-range",
+		message: `Invalid date range. --start ${flags.start} is after --end ${flags.end}.`
+	};
+	if (flags.prevStart !== void 0 && flags.prevEnd !== void 0 && flags.prevStart > flags.prevEnd) return {
+		kind: "inverted-range",
+		message: `Invalid comparison range. --prev-start ${flags.prevStart} is after --prev-end ${flags.prevEnd}.`
+	};
+}
+function parseWindowFlags(flags, defaults, anchor) {
+	const invalid = checkWindowFlags(flags);
+	if (invalid) return err(invalid);
+	const preset = flags.period ? PERIOD_ALIASES[flags.period.toLowerCase()] : void 0;
+	const comparison = flags.vs ? COMPARISON_ALIASES[flags.vs.toLowerCase()] : defaults.comparison;
+	const hasBounds = flags.start !== void 0 || flags.end !== void 0;
+	let window;
+	if (preset === "custom" || hasBounds) {
+		const end = flags.end ?? anchor;
+		const start = flags.start ?? resolveWindow({
+			preset: defaults.preset,
+			anchor: end
+		}).start;
+		if (start > end) return err({
+			kind: "inverted-range",
+			message: `Invalid date range. --start ${start} is after --end ${end}.`
+		});
+		window = resolveWindow({
+			preset: "custom",
+			start,
+			end,
+			comparison
+		});
+	} else window = resolveWindow({
+		preset: preset ?? defaults.preset,
+		anchor,
+		comparison
+	});
+	if (flags.prevStart !== void 0 && flags.prevEnd !== void 0) window = {
+		...window,
+		comparison: {
+			start: flags.prevStart,
+			end: flags.prevEnd
+		}
+	};
+	return ok(window);
+}
+function windowFlagErrorToException(error) {
+	const exception = new Error(error.message);
+	exception.windowFlagError = error;
+	return exception;
+}
+function newestDoneDate(states, tables) {
+	const wanted = tables.length ? tables : [...new Set(states.map((state) => state.table))];
+	if (!wanted.length) return void 0;
+	let anchor;
+	for (const table of wanted) {
+		let newest;
+		for (const state of states) if (state.table === table && state.state === "done" && (state.searchType ?? "web") === "web" && (!newest || state.date > newest)) newest = state.date;
+		if (!newest) return void 0;
+		if (!anchor || newest < anchor) anchor = newest;
+	}
+	return anchor;
+}
+export { COMPARISON_ALIASES, COMPARISON_FLAGS, DEFAULT_WINDOW, PERIOD_ALIASES, PERIOD_FLAGS, checkWindowFlags, newestDoneDate, parseWindowFlags, windowFlagErrorToException };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cli",
   "type": "module",
-  "version": "3.7.0",
+  "version": "4.0.0",
   "description": "CLI for Google Search Console and Bing with hosted or local authentication, data exports, and an MCP server",
   "author": {
     "name": "Harlan Wilton",
@@ -39,28 +39,31 @@
     "skills"
   ],
   "engines": {
-    "node": ">=22"
+    "node": ">=22.13.0"
   },
   "dependencies": {
-    "@clack/prompts": "^1.8.0",
-    "@duckdb/node-api": "1.5.5-r.4",
-    "@gscdump/analysis": "^3.7.0",
-    "@gscdump/engine": "^3.7.0",
-    "@gscdump/engine-gsc-api": "^3.7.0",
-    "@gscdump/sdk": "^3.7.0",
-    "@modelcontextprotocol/sdk": "^1.30.0",
+    "@clack/prompts": "^1.8.1",
+    "@duckdb/node-api": "1.5.5-r.5",
+    "@gscdump/analysis": "^4.0.0",
+    "@gscdump/contracts": "^4.0.0",
+    "@gscdump/engine": "^4.0.0",
+    "@gscdump/engine-gsc-api": "^4.0.0",
+    "@gscdump/sdk": "^4.0.0",
+    "@modelcontextprotocol/sdk": "^1.30.1",
     "citty": "^0.2.2",
     "consola": "^3.4.2",
-    "google-auth-library": "^11.0.2",
-    "gscdump": "^3.7.0",
+    "google-auth-library": "^11.1.0",
+    "gscdump": "^4.0.0",
     "ofetch": "^1.5.1",
-    "open": "^11.0.2",
+    "open": "^11.0.4",
+    "proper-lockfile": "^4.1.2",
     "sitemapd": "^0.2.2",
     "string-width": "^8.2.2",
-    "zod": "^4.6.1"
+    "zod": "^4.6.5"
   },
   "devDependencies": {
-    "vitest": "^5.0.0"
+    "@types/proper-lockfile": "^4.1.4",
+    "vitest": "^5.0.1"
   },
   "scripts": {
     "build": "obuild",