@gscdump/cli 3.8.0 → 4.0.1

package/dist/package.mjs

@@ -1,2 +1,2 @@
-var version = "3.8.0";
+var version = "4.0.1";
 export { version };

package/dist/quota-ledger.mjs

@@ -0,0 +1,370 @@
+import process from "node:process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { classifyError } from "gscdump/errors";
+import { getNextPstMidnight, getPstDate } from "gscdump/dates";
+const QUOTA_CAPS = {
+	searchAnalytics: {
+		perMinute: 1200,
+		scope: "site"
+	},
+	urlInspection: {
+		perDay: 2e3,
+		perMinute: 600,
+		scope: "site"
+	},
+	indexing: {
+		perDay: 200,
+		perMinute: 380,
+		scope: "project"
+	}
+};
+const SHORT_TERM_BLOCK_MS = {
+	perMinute: 6e4,
+	load: 9e5
+};
+function emptyLedgerState() {
+	return {
+		version: 1,
+		usage: []
+	};
+}
+function siteKey(api, site) {
+	return QUOTA_CAPS[api].scope === "project" ? "*" : site;
+}
+function findUsage(state, api, site, day) {
+	return state.usage.find((entry) => entry.api === api && entry.site === site && entry.day === day);
+}
+function withUsage(state, next) {
+	return {
+		version: 1,
+		usage: [...state.usage.filter((entry) => !(entry.api === next.api && entry.site === next.site && entry.day === next.day)), next]
+	};
+}
+function activeBlock(state, api, site, now) {
+	let block;
+	for (const entry of state.usage) {
+		if (entry.api !== api || entry.site !== site || entry.blockedUntil === void 0 || entry.blockedUntil <= now) continue;
+		if (!block || entry.blockedUntil > block.until) block = {
+			until: entry.blockedUntil,
+			reason: entry.reason ?? "quota"
+		};
+	}
+	return block;
+}
+function reserveQuota(state, input) {
+	const site = siteKey(input.api, input.site);
+	const now = input.now.getTime();
+	const block = activeBlock(state, input.api, site, now);
+	if (block) return {
+		decision: {
+			kind: "exhausted",
+			resetsAt: block.until,
+			reason: block.reason
+		},
+		state
+	};
+	const day = getPstDate(input.now);
+	const usage = findUsage(state, input.api, site, day) ?? {
+		api: input.api,
+		site,
+		day,
+		used: 0
+	};
+	const cap = QUOTA_CAPS[input.api].perDay;
+	const left = cap === void 0 ? input.n : Math.max(0, cap - usage.used);
+	if (left <= 0) return {
+		decision: {
+			kind: "exhausted",
+			resetsAt: getNextPstMidnight(input.now),
+			reason: `${cap} calls a day`
+		},
+		state
+	};
+	const n = Math.min(input.n, left);
+	return {
+		decision: n < input.n ? {
+			kind: "partial",
+			n,
+			resetsAt: getNextPstMidnight(input.now)
+		} : {
+			kind: "granted",
+			n
+		},
+		state: withUsage(state, {
+			...usage,
+			used: usage.used + n
+		})
+	};
+}
+function blockUntil(api, reason, now) {
+	if (api === "searchAnalytics") {
+		const perMinute = /qps|per minute|rate limit|ratelimit/i.test(reason);
+		return now.getTime() + SHORT_TERM_BLOCK_MS[perMinute ? "perMinute" : "load"];
+	}
+	if (/per minute|qps/i.test(reason)) return now.getTime() + SHORT_TERM_BLOCK_MS.perMinute;
+	return getNextPstMidnight(now);
+}
+function recordQuotaOutcome(state, input) {
+	const site = siteKey(input.api, input.site);
+	const day = getPstDate(input.now);
+	const usage = findUsage(state, input.api, site, day) ?? {
+		api: input.api,
+		site,
+		day,
+		used: 0
+	};
+	if (input.outcome.kind === "unused") return withUsage(state, {
+		...usage,
+		used: Math.max(0, usage.used - input.outcome.n)
+	});
+	return withUsage(state, {
+		...usage,
+		blockedUntil: blockUntil(input.api, input.outcome.reason, input.now),
+		reason: input.outcome.reason
+	});
+}
+function quotaStatus(state, input) {
+	const site = siteKey(input.api, input.site);
+	const usage = findUsage(state, input.api, site, getPstDate(input.now));
+	const block = activeBlock(state, input.api, site, input.now.getTime());
+	return {
+		api: input.api,
+		site,
+		used: usage?.used ?? 0,
+		...QUOTA_CAPS[input.api].perDay !== void 0 ? { perDay: QUOTA_CAPS[input.api].perDay } : {},
+		...block ? { blocked: block } : {}
+	};
+}
+function addUsage(state, input) {
+	const site = siteKey(input.api, input.site);
+	const day = getPstDate(input.now);
+	const usage = findUsage(state, input.api, site, day) ?? {
+		api: input.api,
+		site,
+		day,
+		used: 0
+	};
+	return withUsage(state, {
+		...usage,
+		used: usage.used + input.n
+	});
+}
+function pruneLedger(state, now) {
+	const today = getPstDate(now);
+	const keepFrom = (/* @__PURE__ */ new Date(Date.parse(`${today}T00:00:00Z`) - 864e5)).toISOString().slice(0, 10);
+	const at = now.getTime();
+	return {
+		version: 1,
+		usage: state.usage.filter((entry) => entry.day >= keepFrom || (entry.blockedUntil ?? 0) > at)
+	};
+}
+function mergeLedger(base, delta) {
+	let merged = base;
+	for (const entry of delta.usage) {
+		const prior = findUsage(merged, entry.api, entry.site, entry.day);
+		const blockedUntil = Math.max(prior?.blockedUntil ?? 0, entry.blockedUntil ?? 0);
+		const reason = (entry.blockedUntil ?? 0) >= (prior?.blockedUntil ?? 0) ? entry.reason : prior?.reason;
+		merged = withUsage(merged, {
+			api: entry.api,
+			site: entry.site,
+			day: entry.day,
+			used: Math.max(0, (prior?.used ?? 0) + entry.used),
+			...blockedUntil > 0 ? { blockedUntil } : {},
+			...reason ? { reason } : {}
+		});
+	}
+	return merged;
+}
+function parseQuotaRefusal(error) {
+	if (classifyError(error).kind !== "rate-limited") return void 0;
+	return googleErrorMessage(error);
+}
+function googleErrorMessage(error) {
+	const message = (error?.data)?.error?.message;
+	const status = error?.statusCode ?? error?.status;
+	if (typeof message === "string" && message.length > 0) return typeof status === "number" ? `${status} ${message}` : message;
+	return error instanceof Error ? error.message : String(error);
+}
+function quotaLedgerPath(dataDir) {
+	return path.join(dataDir, "quota-ledger.json");
+}
+function parseLedgerState(value) {
+	const usage = value?.usage;
+	if (value?.version !== 1 || !Array.isArray(usage)) throw new Error("Quota ledger has an unknown format. Delete quota-ledger.json in the data dir to reset it.");
+	return {
+		version: 1,
+		usage: usage.filter((entry) => typeof entry === "object" && entry !== null && typeof entry.api === "string" && entry.api in QUOTA_CAPS && typeof entry.site === "string" && typeof entry.day === "string" && typeof entry.used === "number")
+	};
+}
+async function readLedgerState(file) {
+	const body = await fs.readFile(file, "utf8").catch((error) => {
+		if (error.code === "ENOENT") return void 0;
+		throw error;
+	});
+	return body === void 0 ? emptyLedgerState() : parseLedgerState(JSON.parse(body));
+}
+async function writeLedgerState(file, state) {
+	await fs.mkdir(path.dirname(file), { recursive: true });
+	const temporary = `${file}.${process.pid}.${Date.now()}.tmp`;
+	await fs.writeFile(temporary, JSON.stringify(state), "utf8");
+	await fs.rename(temporary, file);
+}
+async function openQuotaLedger(opts) {
+	const file = quotaLedgerPath(opts.dataDir);
+	const now = opts.now ?? (() => /* @__PURE__ */ new Date());
+	let state = pruneLedger(await readLedgerState(file), now());
+	let delta = emptyLedgerState();
+	let flushes = Promise.resolve();
+	const flush = () => {
+		const next = flushes.then(async () => {
+			state = pruneLedger(mergeLedger(await readLedgerState(file), delta), now());
+			delta = emptyLedgerState();
+			await writeLedgerState(file, state);
+		});
+		flushes = next.catch(() => {});
+		return next;
+	};
+	return {
+		reserve(api, site, n) {
+			const at = now();
+			const result = reserveQuota(state, {
+				api,
+				site,
+				n,
+				now: at
+			});
+			state = result.state;
+			if (result.decision.kind !== "exhausted") delta = addUsage(delta, {
+				api,
+				site,
+				n: result.decision.n,
+				now: at
+			});
+			return result.decision;
+		},
+		record(api, site, outcome) {
+			const at = now();
+			state = recordQuotaOutcome(state, {
+				api,
+				site,
+				outcome,
+				now: at
+			});
+			delta = outcome.kind === "unused" ? addUsage(delta, {
+				api,
+				site,
+				n: -outcome.n,
+				now: at
+			}) : recordQuotaOutcome(delta, {
+				api,
+				site,
+				outcome,
+				now: at
+			});
+		},
+		status(api, site) {
+			return quotaStatus(state, {
+				api,
+				site,
+				now: now()
+			});
+		},
+		flush
+	};
+}
+const SEARCH_ANALYTICS_PATH_RE = /^\/webmasters\/v3\/sites\/([^/]+)\/searchAnalytics\/query$/;
+function quotaTarget(url, body) {
+	const parsed = new URL(url);
+	if (parsed.origin === "https://searchconsole.googleapis.com") {
+		const match = parsed.pathname.match(SEARCH_ANALYTICS_PATH_RE);
+		if (match) return {
+			api: "searchAnalytics",
+			site: decodeURIComponent(match[1])
+		};
+		if (parsed.pathname === "/v1/urlInspection/index:inspect") {
+			const fields = typeof body === "string" ? JSON.parse(body) : body;
+			return typeof fields?.siteUrl === "string" ? {
+				api: "urlInspection",
+				site: fields.siteUrl
+			} : void 0;
+		}
+	}
+	if (parsed.origin === "https://indexing.googleapis.com" && parsed.pathname === "/v3/urlNotifications:publish") return {
+		api: "indexing",
+		site: "*"
+	};
+}
+function quotaStopError(stop) {
+	return Object.assign(new Error(formatQuotaStop(stop)), {
+		name: "QuotaStop",
+		quotaStop: stop
+	});
+}
+function quotaStopOf(error) {
+	return error?.quotaStop;
+}
+const API_LABELS = {
+	searchAnalytics: "Search Analytics",
+	urlInspection: "URL Inspection",
+	indexing: "Indexing API"
+};
+function formatQuotaStop(stop, now = /* @__PURE__ */ new Date()) {
+	const scope = stop.site === "*" ? "this Google Cloud project" : stop.site;
+	const at = new Date(stop.resetsAt);
+	const sameDay = at.toDateString() === now.toDateString();
+	const when = at.toLocaleString("en-US", sameDay ? {
+		hour: "numeric",
+		minute: "2-digit"
+	} : {
+		month: "short",
+		day: "numeric",
+		hour: "numeric",
+		minute: "2-digit"
+	});
+	return `The Google ${API_LABELS[stop.api]} quota for ${scope} is used up (${stop.reason}). It resets at ${when}. Run the command again after that.`;
+}
+function requestUrl(request) {
+	return typeof request === "string" ? request : request instanceof URL ? request.href : request.url;
+}
+function asArray(hooks) {
+	return hooks === void 0 ? [] : Array.isArray(hooks) ? hooks : [hooks];
+}
+function quotaFetchOptions(ledger, base = {}) {
+	return {
+		...base,
+		onRequest: [async (context) => {
+			const target = quotaTarget(requestUrl(context.request), context.options.body);
+			if (!target) return;
+			const decision = ledger.reserve(target.api, target.site, 1);
+			if (decision.kind === "exhausted") throw quotaStopError({
+				...target,
+				reason: decision.reason,
+				resetsAt: decision.resetsAt
+			});
+			await ledger.flush();
+		}, ...asArray(base.onRequest)],
+		onResponseError: [...asArray(base.onResponseError), async (context) => {
+			const target = quotaTarget(requestUrl(context.request), context.options.body);
+			if (!target) return;
+			const refusal = parseQuotaRefusal({
+				statusCode: context.response.status,
+				data: context.response._data,
+				message: context.error?.message ?? ""
+			});
+			if (!refusal) return;
+			ledger.record(target.api, target.site, {
+				kind: "refused",
+				reason: refusal
+			});
+			await ledger.flush();
+			const block = ledger.status(target.api, target.site).blocked;
+			throw quotaStopError({
+				...target,
+				reason: refusal,
+				resetsAt: block?.until ?? Date.now()
+			});
+		}]
+	};
+}
+export { QUOTA_CAPS, blockUntil, emptyLedgerState, formatQuotaStop, googleErrorMessage, mergeLedger, openQuotaLedger, parseQuotaRefusal, pruneLedger, quotaFetchOptions, quotaLedgerPath, quotaStatus, quotaStopError, quotaStopOf, quotaTarget, readLedgerState, recordQuotaOutcome, reserveQuota };

package/dist/render/analysis.mjs

@@ -1,6 +1,7 @@
 import { renderTable, textLines } from "./layout.mjs";
 import { finite, formatMetric, metricLabel } from "./metrics.mjs";
 import { barColumn, calendarBuckets, renderBars, renderShare, renderSparklines } from "./charts.mjs";
+import { MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
 import { parseGscSiteUrl } from "gscdump";
 function asRecord(value) {
 	return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
@@ -24,6 +25,10 @@ function columnsFor(rows, keys) {
 		}
 	}));
 }
+function coverageWarning(coverage) {
+	if (coverage?.kind !== "truncated") return void 0;
+	return `! Partial data: a fetch stopped at ${coverage.fetched.toLocaleString("en-US")} rows. To read more rows, pass --fetch-budget (max ${MAX_FETCH_BUDGET}).`;
+}
 function renderAnalysis(result, context, options) {
 	const rows = result.results;
 	const meta = result.meta;
@@ -31,6 +36,8 @@ function renderAnalysis(result, context, options) {
 	const end = typeof meta.endDate === "string" ? meta.endDate : context.end;
 	const lines = [...textLines(`${parseGscSiteUrl(context.site).hostname} / ${context.id}`, options, "accent"), ...textLines(`${start} to ${end}`, options, "muted")];
 	if (context.previous) lines.push(...textLines(`vs ${context.previous.start} to ${context.previous.end}`, options, "muted"));
+	const warning = coverageWarning(meta.coverage);
+	if (warning) lines.push(...textLines(warning, options, "warning"));
 	lines.push("");
 	if (!rows.length) {
 		lines.push(...textLines("No results for this period.", options));
@@ -95,4 +102,4 @@ function renderAnalysis(result, context, options) {
 	if (total !== null && total > rows.length) lines.push("", ...textLines(`${rows.length} of ${formatMetric("clicks", total)} rows`, options, "muted"));
 	return lines.join("\n");
 }
-export { asRecord, columnsFor, renderAnalysis, rowLabel };
+export { asRecord, columnsFor, coverageWarning, renderAnalysis, rowLabel };

package/dist/render/report.mjs

@@ -1,11 +1,18 @@
 import { renderTable, textLines } from "./layout.mjs";
 import { barColumn, renderMetrics } from "./charts.mjs";
-import { columnsFor } from "./analysis.mjs";
+import { columnsFor, coverageWarning } from "./analysis.mjs";
 import { parseGscSiteUrl } from "gscdump";
 function renderCliReport(report, options) {
 	const lines = [...textLines(`${parseGscSiteUrl(report.site).hostname} / ${report.id}`, options, "accent"), ...textLines(`${report.window.start} to ${report.window.end}`, options, "muted")];
 	if (report.window.comparison) lines.push(...textLines(`vs ${report.window.comparison.start} to ${report.window.comparison.end}`, options, "muted"));
 	if (report.meta.degraded) lines.push(...textLines(`! Unavailable: ${report.meta.steps.filter((step) => step.status === "error").map((step) => step.key).join(", ")}`, options, "warning"));
+	const truncated = report.meta.steps.filter((step) => step.coverage?.kind === "truncated");
+	const largest = truncated.reduce((max, step) => Math.max(max, step.coverage?.kind === "truncated" ? step.coverage.fetched : 0), 0);
+	const warning = coverageWarning(truncated.length ? {
+		kind: "truncated",
+		fetched: largest
+	} : void 0);
+	if (warning) lines.push(...textLines(`${warning} Steps: ${truncated.map((step) => step.key).join(", ")}.`, options, "warning"));
 	const sections = report.sections.filter((section) => section.findings.length || section.coverage === "partial");
 	for (const section of sections) {
 		const status = section.coverage === "partial" ? " (partial)" : ["info", "low"].includes(section.severity) ? "" : ` [${section.severity}]`;

package/dist/request-pacer.mjs

@@ -0,0 +1,29 @@
+function createRequestPacer(opts) {
+	const now = opts.now ?? Date.now;
+	const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+	const spacing = 6e4 / opts.perMinute;
+	let nextStart = 0;
+	let inFlight = 0;
+	const waiting = [];
+	async function acquire() {
+		if (inFlight >= opts.maxInFlight) await new Promise((resolve) => waiting.push(resolve));
+		inFlight++;
+		const slot = Math.max(now(), nextStart);
+		nextStart = slot + spacing;
+		const wait = slot - now();
+		if (wait > 0) await sleep(wait);
+	}
+	function release() {
+		inFlight--;
+		waiting.shift()?.();
+	}
+	return { async run(task) {
+		await acquire();
+		try {
+			return await task();
+		} finally {
+			release();
+		}
+	} };
+}
+export { createRequestPacer };