@gscdump/cli 3.8.0 → 4.0.1

package/dist/local-entities.mjs

@@ -0,0 +1,658 @@
+import { blockUntil, googleErrorMessage, parseQuotaRefusal } from "./quota-ledger.mjs";
+import { runWithConcurrency } from "./utils.mjs";
+import { latestByUrl, planInspections, recentMonths, scheduleOf, toInspectionRecord } from "./inspection-record.mjs";
+import { createIndexingMetadataStore, createInspectionStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, hashUrl, parseSitemapFeedIdentity } from "@gscdump/engine/entities";
+async function loadInspectionHistory(dataSource, ctx) {
+	const inspector = createInspectionStore({ dataSource });
+	const months = await inspector.listHistoryMonths(ctx);
+	const records = [];
+	for (const month of months) {
+		const shard = await inspector.loadHistory(ctx, month);
+		if (shard) records.push(...shard.records);
+	}
+	return records;
+}
+async function loadInspectionState(dataSource, ctx, now) {
+	const inspector = createInspectionStore({ dataSource });
+	const indexed = await inspector.loadMaterialized(ctx);
+	if (!indexed) {
+		const history = await loadInspectionHistory(dataSource, ctx);
+		return {
+			latest: latestByUrl(history),
+			recent: history
+		};
+	}
+	const recent = [];
+	for (const month of recentMonths(now)) {
+		const shard = await inspector.loadHistory(ctx, month);
+		if (shard) recent.push(...shard.records);
+	}
+	return {
+		latest: latestByUrl([...indexed.map(fromParquetRow), ...recent]),
+		recent
+	};
+}
+function fromParquetRow(row) {
+	const record = {
+		url: row.url,
+		inspectedAt: row.inspectedAt
+	};
+	for (const field of [
+		"indexStatus",
+		"lastCrawlTime",
+		"googleCanonical",
+		"userCanonical",
+		"coverageState",
+		"robotsTxtState",
+		"indexingState",
+		"pageFetchState",
+		"mobileUsabilityVerdict",
+		"richResultsVerdict"
+	]) {
+		const value = row[field];
+		if (value != null) record[field] = value;
+	}
+	if (row.scheduleNextAt != null && row.scheduleConsecutiveUnchanged != null && row.schedulePolicyVersion != null) record.raw = { schedule: {
+		nextAt: row.scheduleNextAt,
+		consecutiveUnchanged: row.scheduleConsecutiveUnchanged,
+		policyVersion: row.schedulePolicyVersion
+	} };
+	return record;
+}
+function toParquetRow(record) {
+	const schedule = scheduleOf(record);
+	return {
+		urlHash: hashUrl(record.url),
+		url: record.url,
+		inspectedAt: record.inspectedAt,
+		indexStatus: record.indexStatus ?? null,
+		lastCrawlTime: record.lastCrawlTime ?? null,
+		googleCanonical: record.googleCanonical ?? null,
+		userCanonical: record.userCanonical ?? null,
+		coverageState: record.coverageState ?? null,
+		robotsTxtState: record.robotsTxtState ?? null,
+		indexingState: record.indexingState ?? null,
+		pageFetchState: record.pageFetchState ?? null,
+		mobileUsabilityVerdict: record.mobileUsabilityVerdict ?? null,
+		richResultsVerdict: record.richResultsVerdict ?? null,
+		scheduleNextAt: schedule.nextAt,
+		scheduleConsecutiveUnchanged: schedule.consecutiveUnchanged,
+		schedulePolicyVersion: schedule.policyVersion
+	};
+}
+async function appendInspections(dataSource, ctx, records) {
+	if (records.length > 0) await createInspectionStore({ dataSource }).appendHistory(ctx, records);
+}
+async function materializeInspectionIndex(dataSource, ctx, latest, records) {
+	const merged = latestByUrl([...latest.values(), ...records]);
+	const result = await createInspectionStore({ dataSource }).materialize(ctx, [...merged.values()].map(toParquetRow));
+	return {
+		indexKey: result.key,
+		indexRows: result.rowCount
+	};
+}
+async function recordInspections(dataSource, ctx, latest, records) {
+	if (records.length === 0) return void 0;
+	await appendInspections(dataSource, ctx, records);
+	return materializeInspectionIndex(dataSource, ctx, latest, records);
+}
+function urlInProperty(siteUrl, url) {
+	let parsed;
+	try {
+		parsed = new URL(url);
+	} catch {
+		return false;
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return false;
+	if (siteUrl.startsWith("sc-domain:")) {
+		const domain = siteUrl.slice(10).toLowerCase();
+		const host = parsed.hostname.toLowerCase();
+		return host === domain || host.endsWith(`.${domain}`);
+	}
+	return url.startsWith(siteUrl);
+}
+function resolvePagePaths(paths, siteUrl, knownUrls) {
+	let origin;
+	if (siteUrl.startsWith("sc-domain:")) {
+		const counts = /* @__PURE__ */ new Map();
+		for (const url of knownUrls) {
+			if (!urlInProperty(siteUrl, url)) continue;
+			const o = new URL(url).origin;
+			counts.set(o, (counts.get(o) ?? 0) + 1);
+		}
+		origin = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] ?? `https://${siteUrl.slice(10)}`;
+	}
+	const out = [];
+	for (const value of paths) try {
+		out.push(new URL(value, origin ?? siteUrl).toString());
+	} catch {}
+	return out;
+}
+const INSPECT_PAGE_CANDIDATES = 5e3;
+async function topPagePaths(store, siteUrl) {
+	if ((await store.engine.listLive({
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl),
+		table: "pages"
+	})).length === 0) return [];
+	const { rows } = await store.runRawSql({
+		sql: `SELECT url, SUM(impressions) AS impressions FROM read_parquet({{FILES}}, union_by_name = true) GROUP BY url ORDER BY impressions DESC LIMIT ${INSPECT_PAGE_CANDIDATES}`,
+		siteUrl,
+		table: "pages"
+	});
+	return rows.map((row) => String(row.url));
+}
+async function inspectionCandidates(store, siteUrl) {
+	const ctx = {
+		userId: store.userId,
+		siteId: store.siteIdFor(siteUrl)
+	};
+	const sitemapUrls = await loadSitemapGenerationUrls(store.dataSource, ctx);
+	const trafficUrls = resolvePagePaths(await topPagePaths(store, siteUrl), siteUrl, sitemapUrls);
+	const rank = new Map(trafficUrls.map((url, i) => [url, i]));
+	const sitemapFirst = [...sitemapUrls].sort((a, b) => (rank.get(a) ?? Infinity) - (rank.get(b) ?? Infinity));
+	const listed = new Set(sitemapUrls);
+	return [.../* @__PURE__ */ new Set([...sitemapFirst, ...trafficUrls.filter((url) => !listed.has(url))])].filter((url) => urlInProperty(siteUrl, url));
+}
+const INSPECTION_CHUNK = 20;
+async function syncInspections(deps) {
+	const state = await loadInspectionState(deps.dataSource, deps.ctx, deps.now());
+	let latest = state.latest;
+	const candidates = deps.candidates.filter((url) => urlInProperty(deps.siteUrl, url));
+	const plan = planInspections({
+		candidates,
+		latest,
+		recent: state.recent,
+		now: deps.now(),
+		limit: deps.limit
+	});
+	if (plan.urls.length === 0) return {
+		_tag: "nothing_due",
+		candidates: candidates.length,
+		quotaLeft: plan.quotaLeft
+	};
+	const failures = [];
+	let inspected = 0;
+	let done = 0;
+	let stopped;
+	const queue = [...plan.urls];
+	while (queue.length > 0 && !stopped) {
+		const decision = deps.ledger.reserve("urlInspection", deps.siteUrl, Math.min(INSPECTION_CHUNK, queue.length));
+		if (decision.kind === "exhausted") {
+			stopped = {
+				reason: decision.reason,
+				resetsAt: decision.resetsAt
+			};
+			break;
+		}
+		const chunk = queue.splice(0, decision.n);
+		const records = [];
+		let unused = 0;
+		await runWithConcurrency(chunk, deps.concurrency, async (url) => {
+			if (stopped) {
+				unused++;
+				return;
+			}
+			const response = await deps.client.inspect(deps.siteUrl, url).catch((error) => ({ error }));
+			if ("error" in response) {
+				const refusal = parseQuotaRefusal(response.error);
+				if (refusal) {
+					deps.ledger.record("urlInspection", deps.siteUrl, {
+						kind: "refused",
+						reason: refusal
+					});
+					stopped ??= {
+						reason: refusal,
+						resetsAt: blockUntil("urlInspection", refusal, deps.now())
+					};
+					unused++;
+					return;
+				}
+				failures.push({
+					url,
+					error: googleErrorMessage(response.error)
+				});
+			} else records.push(toInspectionRecord({
+				url,
+				result: response.inspectionResult,
+				inspectedAt: deps.now(),
+				previous: latest.get(url)
+			}));
+			done++;
+			deps.onProgress?.(done, plan.urls.length);
+		});
+		if (unused > 0) deps.ledger.record("urlInspection", deps.siteUrl, {
+			kind: "unused",
+			n: unused
+		});
+		await recordInspections(deps.dataSource, deps.ctx, latest, records);
+		latest = latestByUrl([...latest.values(), ...records]);
+		inspected += records.length;
+	}
+	const notRun = plan.urls.length - done;
+	if (done === 0 && stopped) return {
+		_tag: "quota_exhausted",
+		reason: stopped.reason,
+		resetsAt: stopped.resetsAt,
+		deferred: plan.deferred + notRun
+	};
+	return {
+		_tag: "inspected",
+		inspected,
+		failed: failures.length,
+		failures,
+		deferred: plan.deferred + notRun,
+		quotaLeft: Math.max(0, plan.quotaLeft - done),
+		...stopped ? { stopped } : {}
+	};
+}
+function toCount(value) {
+	const n = Number(value);
+	return Number.isFinite(n) ? n : 0;
+}
+function toSitemapListEntry(sitemap) {
+	if (!sitemap.path) return void 0;
+	return {
+		path: sitemap.path,
+		type: sitemap.type ?? null,
+		isPending: sitemap.isPending ?? false,
+		isSitemapsIndex: sitemap.isSitemapsIndex ?? false,
+		lastSubmitted: sitemap.lastSubmitted ?? null,
+		lastDownloaded: sitemap.lastDownloaded ?? null,
+		warnings: toCount(sitemap.warnings),
+		errors: toCount(sitemap.errors),
+		contents: (sitemap.contents ?? []).map((content) => ({
+			type: content.type ?? "unknown",
+			submitted: toCount(content.submitted),
+			indexed: content.indexed == null ? null : toCount(content.indexed)
+		}))
+	};
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+async function syncSitemaps(deps) {
+	const listed = await deps.client.sitemaps.list(deps.siteUrl).catch((error) => new Error(errorMessage(error)));
+	if (listed instanceof Error) return {
+		_tag: "failed",
+		reason: `Search Console sitemap list failed: ${listed.message}`
+	};
+	const fetchedAt = deps.now();
+	const sitemaps = listed.flatMap((sitemap) => toSitemapListEntry(sitemap) ?? []);
+	const doc = {
+		version: 1,
+		fetchedAt: fetchedAt.toISOString(),
+		sitemaps
+	};
+	await createSitemapListStore({ dataSource: deps.dataSource }).save(deps.ctx, doc);
+	const store = createSitemapStore({
+		dataSource: deps.dataSource,
+		withMutation: (_ctx, effect) => effect(),
+		now: () => deps.now().getTime()
+	});
+	const current = await store.getSitemapGeneration(deps.ctx);
+	const published = new Set(current._tag === "available" ? Object.values(current.manifest.feeds).map((feed) => feed.feedpath) : []);
+	if (sitemaps.length === 0 && published.size === 0) return {
+		_tag: "saved",
+		sitemaps: 0,
+		urls: 0,
+		feeds: [],
+		generation: "unchanged"
+	};
+	const generation = {
+		_tag: "complete",
+		id: deps.generationId(),
+		observedAt: fetchedAt.getTime()
+	};
+	const feeds = [];
+	const staged = [];
+	let urls = 0;
+	let keepReason;
+	for (const sitemap of sitemaps) {
+		const loaded = await deps.loadFeed(sitemap.path);
+		if (loaded._tag === "error") {
+			feeds.push({
+				_tag: "unreachable",
+				path: sitemap.path,
+				message: loaded.message
+			});
+			const identity = parseSitemapFeedIdentity(sitemap.path);
+			if (identity._tag === "ok" && published.has(identity.url)) {
+				keepReason = `${sitemap.path} could not be read: ${loaded.message}`;
+				break;
+			}
+			continue;
+		}
+		const result = await store.stageSitemapGenerationFeed(deps.ctx, generation, sitemap.path, loaded.entries);
+		if (result._tag !== "staged") {
+			feeds.push({
+				_tag: "unreachable",
+				path: sitemap.path,
+				message: `not stored (${result._tag}: ${result.reason})`
+			});
+			continue;
+		}
+		staged.push(sitemap.path);
+		urls += loaded.entries.length;
+		feeds.push({
+			_tag: "read",
+			path: sitemap.path,
+			urls: loaded.entries.length,
+			complete: loaded.complete
+		});
+	}
+	if (keepReason) {
+		await store.abortSitemapGeneration(deps.ctx, generation);
+		return {
+			_tag: "list_only",
+			sitemaps: sitemaps.length,
+			feeds,
+			reason: keepReason
+		};
+	}
+	const finalized = await store.finalizeSitemapGeneration(deps.ctx, generation, {
+		_tag: "complete",
+		expectedFeedpaths: staged
+	});
+	if (finalized._tag === "published" || finalized._tag === "unchanged") return {
+		_tag: "saved",
+		sitemaps: sitemaps.length,
+		urls,
+		feeds,
+		generation: finalized._tag
+	};
+	await store.abortSitemapGeneration(deps.ctx, generation);
+	return {
+		_tag: "list_only",
+		sitemaps: sitemaps.length,
+		feeds,
+		reason: `sitemap URLs not stored: ${finalized._tag}`
+	};
+}
+async function loadSitemapGenerationUrls(dataSource, ctx) {
+	const result = await createSitemapReadStore({ dataSource }).iterateSitemapGenerationUrls(ctx);
+	if (result._tag !== "iterator") return [];
+	const urls = [];
+	for await (const record of result.items) urls.push(record.loc);
+	return urls;
+}
+const ENTITY_DATASETS = [
+	"inspections",
+	"inspection_history",
+	"sitemaps",
+	"sitemap_urls",
+	"indexing_metadata"
+];
+const INSPECTION_COLUMNS = [
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspected_at",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "index_status",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverage_state",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexing_state",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "page_fetch_state",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robots_txt_state",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "last_crawl_time",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "google_canonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "user_canonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobile_usability_verdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "rich_results_verdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "next_check_at",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "raw",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+const SITEMAP_COLUMNS = [
+	{
+		name: "path",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "type",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "is_pending",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "is_sitemaps_index",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "last_submitted",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "last_downloaded",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "warnings",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "errors",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "submitted",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "contents",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "fetched_at",
+		type: "VARCHAR",
+		nullable: false
+	}
+];
+const SITEMAP_URL_COLUMNS = [
+	{
+		name: "sitemap_path",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "VARCHAR",
+		nullable: false
+	}
+];
+const INDEXING_METADATA_COLUMNS = [
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "captured_at",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "latest_update_at",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "latest_remove_at",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+function inspectionRow(record) {
+	const nextAt = scheduleOf(record).nextAt;
+	return {
+		url: record.url,
+		inspected_at: record.inspectedAt,
+		index_status: record.indexStatus ?? null,
+		coverage_state: record.coverageState ?? null,
+		indexing_state: record.indexingState ?? null,
+		page_fetch_state: record.pageFetchState ?? null,
+		robots_txt_state: record.robotsTxtState ?? null,
+		last_crawl_time: record.lastCrawlTime ?? null,
+		google_canonical: record.googleCanonical ?? null,
+		user_canonical: record.userCanonical ?? null,
+		mobile_usability_verdict: record.mobileUsabilityVerdict ?? null,
+		rich_results_verdict: record.richResultsVerdict ?? null,
+		next_check_at: Number.isFinite(nextAt) && nextAt > 0 ? new Date(nextAt).toISOString() : null,
+		raw: record.raw ? JSON.stringify(record.raw) : null
+	};
+}
+function sitemapRow(entry, fetchedAt) {
+	return {
+		path: entry.path,
+		type: entry.type,
+		is_pending: entry.isPending,
+		is_sitemaps_index: entry.isSitemapsIndex,
+		last_submitted: entry.lastSubmitted,
+		last_downloaded: entry.lastDownloaded,
+		warnings: entry.warnings,
+		errors: entry.errors,
+		submitted: entry.contents.reduce((sum, content) => sum + content.submitted, 0),
+		contents: JSON.stringify(entry.contents),
+		fetched_at: fetchedAt
+	};
+}
+async function sitemapUrlRows(dataSource, ctx) {
+	const result = await createSitemapReadStore({ dataSource }).iterateSitemapGenerationUrls(ctx);
+	if (result._tag !== "iterator") return [];
+	const rows = [];
+	for await (const record of result.items) rows.push({
+		sitemap_path: record.feedpath,
+		url: record.loc,
+		lastmod: record.lastmod ?? null,
+		first_seen_at: new Date(record.firstSeenAt).toISOString(),
+		last_seen_at: new Date(record.lastSeenAt).toISOString()
+	});
+	return rows;
+}
+async function readEntityDatasets(dataSource, ctx, wanted) {
+	const want = new Set(wanted);
+	const out = [];
+	if (want.has("inspections") || want.has("inspection_history")) {
+		const history = (await loadInspectionHistory(dataSource, ctx)).sort((a, b) => a.url.localeCompare(b.url) || a.inspectedAt.localeCompare(b.inspectedAt));
+		if (want.has("inspections")) {
+			const latest = [...latestByUrl(history).values()].sort((a, b) => a.url.localeCompare(b.url));
+			out.push({
+				dataset: "inspections",
+				columns: INSPECTION_COLUMNS,
+				rows: latest.map(inspectionRow)
+			});
+		}
+		if (want.has("inspection_history")) out.push({
+			dataset: "inspection_history",
+			columns: INSPECTION_COLUMNS,
+			rows: history.map(inspectionRow)
+		});
+	}
+	if (want.has("sitemaps")) {
+		const doc = await createSitemapListStore({ dataSource }).load(ctx);
+		out.push({
+			dataset: "sitemaps",
+			columns: SITEMAP_COLUMNS,
+			rows: doc ? doc.sitemaps.map((entry) => sitemapRow(entry, doc.fetchedAt)) : []
+		});
+	}
+	if (want.has("sitemap_urls")) out.push({
+		dataset: "sitemap_urls",
+		columns: SITEMAP_URL_COLUMNS,
+		rows: await sitemapUrlRows(dataSource, ctx)
+	});
+	if (want.has("indexing_metadata")) {
+		const index = await createIndexingMetadataStore({ dataSource }).loadIndex(ctx);
+		out.push({
+			dataset: "indexing_metadata",
+			columns: INDEXING_METADATA_COLUMNS,
+			rows: Object.values(index.records).sort((a, b) => a.url.localeCompare(b.url)).map((record) => ({
+				url: record.url,
+				captured_at: record.capturedAt,
+				latest_update_at: record.latestUpdateAt ?? null,
+				latest_remove_at: record.latestRemoveAt ?? null
+			}))
+		});
+	}
+	return out;
+}
+export { ENTITY_DATASETS, appendInspections, inspectionCandidates, loadInspectionHistory, loadInspectionState, loadSitemapGenerationUrls, materializeInspectionIndex, readEntityDatasets, recordInspections, resolvePagePaths, syncInspections, syncSitemaps, toSitemapListEntry, urlInProperty };