@gscdump/cli 3.8.0 → 4.0.0

package/dist/inspect-urls.mjs

@@ -0,0 +1,76 @@
+import { formatQuotaStop, quotaStopOf } from "./quota-ledger.mjs";
+import { classifyError } from "gscdump/errors";
+import { URL_INSPECTION_QUOTA, describeInspectionError } from "gscdump/indexing";
+function checkInspectionBatch(urls) {
+	if (urls.length <= URL_INSPECTION_QUOTA.perDay) return { kind: "ok" };
+	const perDay = URL_INSPECTION_QUOTA.perDay.toLocaleString("en-US");
+	return {
+		kind: "too-many",
+		message: `You passed ${urls.length.toLocaleString("en-US")} URLs. Google allows ${perDay} URL inspections per day for each property. Inspect at most ${perDay} URLs in one run.`
+	};
+}
+async function inspectUrls(input) {
+	const intervalMs = input.intervalMs ?? 120;
+	const sleep = input.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+	const now = input.now ?? Date.now;
+	const outcomes = [];
+	let lastStart = null;
+	for (let i = 0; i < input.urls.length; i++) {
+		const url = input.urls[i];
+		if (!input.inProperty(url)) {
+			const outcome = {
+				kind: "failed",
+				url,
+				error: "The URL is outside this Site."
+			};
+			outcomes.push(outcome);
+			await input.onOutcome(outcome);
+			continue;
+		}
+		if (lastStart !== null) {
+			const wait = lastStart + intervalMs - now();
+			if (wait > 0) await sleep(wait);
+		}
+		lastStart = now();
+		const settled = await input.inspect(url).then((result) => ({
+			ok: true,
+			result
+		}), (error) => ({
+			ok: false,
+			error
+		}));
+		if (!settled.ok) {
+			const quota = quotaStopOf(settled.error);
+			if (quota) return {
+				outcomes,
+				stopped: {
+					reason: formatQuotaStop(quota),
+					remaining: input.urls.length - i
+				}
+			};
+			if (classifyError(settled.error).kind === "rate-limited") return {
+				outcomes,
+				stopped: {
+					reason: describeInspectionError(settled.error),
+					remaining: input.urls.length - i
+				}
+			};
+		}
+		const outcome = settled.ok ? {
+			kind: "inspected",
+			url,
+			result: settled.result
+		} : {
+			kind: "failed",
+			url,
+			error: describeInspectionError(settled.error)
+		};
+		outcomes.push(outcome);
+		await input.onOutcome(outcome);
+	}
+	return {
+		outcomes,
+		stopped: null
+	};
+}
+export { checkInspectionBatch, inspectUrls };

package/dist/inspection-record.mjs

@@ -0,0 +1,93 @@
+import { inspectionPolicy } from "@gscdump/engine";
+const INSPECTION_QPD_PER_PROPERTY = 2e3;
+const DAY_MS = 864e5;
+const STATE_FIELDS = [
+	"indexStatus",
+	"coverageState",
+	"robotsTxtState",
+	"indexingState",
+	"pageFetchState",
+	"googleCanonical"
+];
+function verdictOf(indexStatus) {
+	return indexStatus === "PASS" || indexStatus === "FAIL" ? indexStatus : "NEUTRAL";
+}
+function scheduleOf(record) {
+	if (record.raw?.schedule) return record.raw.schedule;
+	const at = Date.parse(record.inspectedAt);
+	return inspectionPolicy(verdictOf(record.indexStatus)).initial(Number.isFinite(at) ? at : 0);
+}
+function toInspectionRecord(input) {
+	const { url, result, inspectedAt, previous } = input;
+	const indexStatus = result?.indexStatusResult;
+	const record = {
+		url,
+		inspectedAt: inspectedAt.toISOString(),
+		indexStatus: indexStatus?.verdict ?? void 0,
+		lastCrawlTime: indexStatus?.lastCrawlTime ?? void 0,
+		googleCanonical: indexStatus?.googleCanonical ?? void 0,
+		userCanonical: indexStatus?.userCanonical ?? void 0,
+		coverageState: indexStatus?.coverageState ?? void 0,
+		robotsTxtState: indexStatus?.robotsTxtState ?? void 0,
+		indexingState: indexStatus?.indexingState ?? void 0,
+		pageFetchState: indexStatus?.pageFetchState ?? void 0,
+		mobileUsabilityVerdict: result?.mobileUsabilityResult?.verdict ?? void 0,
+		richResultsVerdict: result?.richResultsResult?.verdict ?? void 0
+	};
+	const policy = inspectionPolicy(verdictOf(record.indexStatus));
+	const at = inspectedAt.getTime();
+	const schedule = previous ? policy.observe(scheduleOf(previous), {
+		changed: STATE_FIELDS.some((field) => (previous[field] ?? null) !== (record[field] ?? null)),
+		at
+	}) : policy.initial(at);
+	record.raw = {
+		...result,
+		schedule
+	};
+	return record;
+}
+function recentMonths(now) {
+	const month = (at) => new Date(at).toISOString().slice(0, 7);
+	return [.../* @__PURE__ */ new Set([month(now.getTime() - DAY_MS), month(now.getTime())])];
+}
+function latestByUrl(records) {
+	const latest = /* @__PURE__ */ new Map();
+	for (const record of records) {
+		const prior = latest.get(record.url);
+		if (!prior || record.inspectedAt > prior.inspectedAt) latest.set(record.url, record);
+	}
+	return latest;
+}
+function planInspections(input) {
+	const now = input.now.getTime();
+	const since = now - DAY_MS;
+	const usedToday = input.recent.filter((record) => {
+		const at = Date.parse(record.inspectedAt);
+		return at > since && at <= now;
+	}).length;
+	const quotaLeft = Math.max(0, INSPECTION_QPD_PER_PROPERTY - usedToday);
+	const latest = input.latest;
+	const fresh = [];
+	const due = [];
+	for (const url of new Set(input.candidates)) {
+		const record = latest.get(url);
+		if (!record) {
+			fresh.push(url);
+			continue;
+		}
+		const schedule = scheduleOf(record);
+		if (inspectionPolicy(verdictOf(record.indexStatus)).isDue(schedule, now)) due.push({
+			url,
+			nextAt: schedule.nextAt
+		});
+	}
+	due.sort((a, b) => a.nextAt - b.nextAt);
+	const ordered = [...fresh, ...due.map((entry) => entry.url)];
+	const budget = Math.max(0, Math.min(input.limit, quotaLeft));
+	return {
+		urls: ordered.slice(0, budget),
+		deferred: Math.max(0, ordered.length - budget),
+		quotaLeft
+	};
+}
+export { INSPECTION_QPD_PER_PROPERTY, latestByUrl, planInspections, recentMonths, scheduleOf, toInspectionRecord };