@gscdump/engine 0.25.13 → 0.26.0

package/dist/entities.mjs

@@ -1,641 +1,2 @@
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
-function inspectionIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
-}
-function emptyTypesKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
-}
-function inspectionParquetKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
-}
-function inspectionHistoryPrefix(ctx, yearMonth) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}`;
-}
-function inspectionHistoryShardKey(ctx, yearMonth, batchId) {
-	return `${inspectionHistoryPrefix(ctx, yearMonth)}/${batchId}.json`;
-}
-function hashUrl(url) {
-	let hi = 2166136261;
-	let lo = 3421674724;
-	for (let i = 0; i < url.length; i++) {
-		const c = url.charCodeAt(i);
-		lo ^= c;
-		const loMul = Math.imul(lo, 435) >>> 0;
-		const carry = Math.floor(lo * 435 / 4294967296);
-		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
-		lo = loMul;
-		hi = hiMul;
-	}
-	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
-}
-const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
-const INSPECTION_PARQUET_COLUMNS = [
-	{
-		name: "urlHash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "inspectedAt",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "indexStatus",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "lastCrawlTime",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "googleCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "userCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "coverageState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "robotsTxtState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "indexingState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "pageFetchState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "mobileUsabilityVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "richResultsVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "scheduleNextAt",
-		type: "BIGINT",
-		nullable: true
-	},
-	{
-		name: "scheduleConsecutiveUnchanged",
-		type: "INTEGER",
-		nullable: true
-	},
-	{
-		name: "schedulePolicyVersion",
-		type: "INTEGER",
-		nullable: true
-	}
-];
-function createInspectionStore(opts) {
-	const ds = opts.dataSource;
-	function shardFor(record) {
-		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
-		return m ? `${m[1]}-${m[2]}` : "unknown";
-	}
-	function randomBatchId() {
-		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
-	}
-	return {
-		async appendHistory(ctx, records, options) {
-			if (records.length === 0) return;
-			const batchId = options?.batchId ?? randomBatchId();
-			const byMonth = /* @__PURE__ */ new Map();
-			for (const r of records) {
-				const month = shardFor(r);
-				if (!byMonth.has(month)) byMonth.set(month, []);
-				byMonth.get(month).push(r);
-			}
-			for (const [yearMonth, batch] of byMonth) {
-				const shard = {
-					version: 1,
-					records: batch
-				};
-				const bytes = new TextEncoder().encode(JSON.stringify(shard));
-				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
-				await ds.write(inspectionHistoryShardKey(ctx, yearMonth, batchId), bytes);
-			}
-		},
-		async loadHistory(ctx, yearMonth) {
-			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
-			if (keys.length === 0) return void 0;
-			const out = [];
-			for (const key of keys) {
-				const bytes = await ds.read(key).catch(() => void 0);
-				if (!bytes) continue;
-				const shard = await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
-					console.warn("[inspection.loadHistory] failed to decode shard", {
-						key,
-						error: err.message
-					});
-				});
-				if (shard?.records) out.push(...shard.records);
-			}
-			return {
-				version: 1,
-				records: out
-			};
-		},
-		async materialize(ctx, rowIter) {
-			const rows = Array.from(rowIter);
-			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-			const bytes = encodeRowsToParquetFlex(rows, {
-				columns: INSPECTION_PARQUET_COLUMNS,
-				sortKey: ["urlHash"]
-			});
-			const key = inspectionParquetKey(ctx);
-			await ds.write(key, bytes);
-			return {
-				key,
-				rowCount: rows.length,
-				bytes: bytes.byteLength
-			};
-		},
-		parquetUri(ctx) {
-			return ds.uri?.(inspectionParquetKey(ctx));
-		}
-	};
-}
-function sitemapIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
-}
-function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
-}
-function sitemapUrlsPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
-}
-function sitemapUrlsIndexPrefix(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
-}
-function sitemapUrlsIndexKey(ctx, feedpathHash) {
-	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
-}
-function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
-	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
-}
-const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
-const URLS_INDEX_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "first_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "last_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "removed_at",
-		type: "BIGINT",
-		nullable: true
-	}
-];
-const URLS_DELTA_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "op",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "at",
-		type: "BIGINT",
-		nullable: false
-	}
-];
-function rowToUrlRecord(row) {
-	return {
-		feedpath: String(row.feedpath),
-		feedpathHash: String(row.feedpath_hash),
-		urlHash: String(row.url_hash),
-		loc: String(row.loc),
-		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-		firstSeenAt: Number(row.first_seen_at),
-		lastSeenAt: Number(row.last_seen_at),
-		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
-	};
-}
-function urlRecordToRow(r) {
-	return {
-		feedpath: r.feedpath,
-		feedpath_hash: r.feedpathHash,
-		url_hash: r.urlHash,
-		loc: r.loc,
-		lastmod: r.lastmod ?? null,
-		first_seen_at: r.firstSeenAt,
-		last_seen_at: r.lastSeenAt,
-		removed_at: r.removedAt ?? null
-	};
-}
-function isoDate(ms) {
-	return new Date(ms).toISOString().slice(0, 10);
-}
-function hashUrlList(urls) {
-	return hashUrl(urls.map((u) => u.loc).sort().join("\n"));
-}
-function createSitemapStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	const now = opts.now ?? (() => Date.now());
-	async function readJson(key) {
-		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => void 0);
-	}
-	async function writeJson(key, value) {
-		await ds.write(key, new TextEncoder().encode(JSON.stringify(value)));
-	}
-	return {
-		async writeSnapshot(ctx, records) {
-			if (records.length === 0) return;
-			const indexKey = sitemapIndexKey(ctx);
-			const index = await readJson(indexKey) ?? {
-				version: 1,
-				records: {}
-			};
-			const stamp = now();
-			for (const r of records) {
-				const h = hash(r.path);
-				index.records[h] = r;
-				await writeJson(sitemapHistoryKey(ctx, h, stamp), {
-					version: 1,
-					path: r.path,
-					capturedAt: r.capturedAt,
-					record: r
-				});
-			}
-			await writeJson(indexKey, index);
-		},
-		async loadIndex(ctx) {
-			return await readJson(sitemapIndexKey(ctx)) ?? {
-				version: 1,
-				records: {}
-			};
-		},
-		async getLatest(ctx, path) {
-			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
-		},
-		async snapshotUrls(ctx, feedpath, urls) {
-			const fpHash = hash(feedpath);
-			const contentHash = hashUrlList(urls);
-			const at = now();
-			const priorByHash = /* @__PURE__ */ new Map();
-			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
-			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
-			if (livePrior.length > 0) {
-				if (hashUrl(livePrior.map((r) => String(r.loc)).sort().join("\n")) === contentHash) return {
-					added: 0,
-					removed: 0,
-					kept: livePrior.length,
-					contentHash,
-					unchanged: true
-				};
-			}
-			const incomingByHash = /* @__PURE__ */ new Map();
-			for (const u of urls) incomingByHash.set(hash(u.loc), u);
-			const deltaRows = [];
-			let added = 0;
-			let removed = 0;
-			let kept = 0;
-			const date = isoDate(at);
-			for (const [urlHash, u] of incomingByHash) {
-				const prev = priorByHash.get(urlHash);
-				if (!prev || prev.removedAt != null) {
-					added++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "added",
-						loc: u.loc,
-						lastmod: u.lastmod ?? null,
-						at
-					});
-				} else kept++;
-			}
-			for (const [urlHash, prev] of priorByHash) {
-				if (prev.removedAt != null) continue;
-				if (!incomingByHash.has(urlHash)) {
-					removed++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "removed",
-						loc: prev.loc,
-						lastmod: prev.lastmod ?? null,
-						at
-					});
-				}
-			}
-			if (deltaRows.length > 0) {
-				const bytes = encodeRowsToParquetFlex(deltaRows, {
-					columns: URLS_DELTA_COLUMNS,
-					sortKey: ["url_hash"]
-				});
-				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
-			}
-			return {
-				added,
-				removed,
-				kept,
-				contentHash,
-				unchanged: false
-			};
-		},
-		async *loadUrls(ctx, feedpath, opts) {
-			const fpHash = hash(feedpath);
-			const includeRemoved = opts?.includeRemoved ?? false;
-			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx, fpHash)).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
-			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
-			const live = /* @__PURE__ */ new Map();
-			const removedMap = /* @__PURE__ */ new Map();
-			for (const row of indexRows) {
-				const rec = rowToUrlRecord(row);
-				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
-				else live.set(rec.urlHash, rec);
-			}
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m || m[2] !== fpHash) continue;
-				const dBytes = await ds.read(key).catch(() => void 0);
-				if (!dBytes) continue;
-				const dRows = await decodeParquetToRows(dBytes);
-				for (const r of dRows) {
-					const op = String(r.op);
-					const urlHash = String(r.url_hash);
-					const at = Number(r.at);
-					if (op === "added") {
-						const prev = live.get(urlHash) ?? removedMap.get(urlHash);
-						removedMap.delete(urlHash);
-						live.set(urlHash, {
-							feedpath,
-							feedpathHash: fpHash,
-							urlHash,
-							loc: String(r.loc),
-							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-							firstSeenAt: prev?.firstSeenAt ?? at,
-							lastSeenAt: at
-						});
-					} else if (op === "removed") {
-						const prev = live.get(urlHash);
-						live.delete(urlHash);
-						if (prev) removedMap.set(urlHash, {
-							...prev,
-							removedAt: at
-						});
-					}
-				}
-			}
-			for (const rec of live.values()) yield rec;
-			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
-		},
-		async *loadDeltas(ctx, dateRange) {
-			const from = dateRange?.from;
-			const to = dateRange?.to;
-			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
-			for (const key of keys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const date = m[1];
-				if (from && date < from) continue;
-				if (to && date > to) continue;
-				const bytes = await ds.read(key).catch(() => void 0);
-				if (!bytes) continue;
-				const rows = await decodeParquetToRows(bytes);
-				for (const r of rows) {
-					const op = String(r.op);
-					if (op !== "added" && op !== "removed") continue;
-					yield {
-						feedpath: String(r.feedpath),
-						feedpathHash: String(r.feedpath_hash),
-						urlHash: String(r.url_hash),
-						op,
-						loc: String(r.loc),
-						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-						at: Number(r.at)
-					};
-				}
-			}
-		},
-		async compactUrls(ctx) {
-			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
-			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const list = deltasByFeed.get(m[2]) ?? [];
-				list.push(key);
-				deltasByFeed.set(m[2], list);
-			}
-			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
-				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const indexBytes = await ds.read(indexKey).catch(() => void 0);
-				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
-				const live = /* @__PURE__ */ new Map();
-				const removed = /* @__PURE__ */ new Map();
-				for (const row of indexRows) {
-					const rec = rowToUrlRecord(row);
-					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
-					else live.set(rec.urlHash, rec);
-				}
-				const consumed = [];
-				for (const key of feedDeltaKeys.sort()) {
-					const bytes = await ds.read(key).catch(() => void 0);
-					if (!bytes) continue;
-					consumed.push(key);
-					const rows = await decodeParquetToRows(bytes);
-					for (const r of rows) {
-						const urlHash = String(r.url_hash);
-						const at = Number(r.at);
-						const op = String(r.op);
-						if (op === "added") {
-							const prev = live.get(urlHash) ?? removed.get(urlHash);
-							removed.delete(urlHash);
-							live.set(urlHash, {
-								feedpath: String(r.feedpath),
-								feedpathHash: fpHash,
-								urlHash,
-								loc: String(r.loc),
-								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-								firstSeenAt: prev?.firstSeenAt ?? at,
-								lastSeenAt: at
-							});
-						} else if (op === "removed") {
-							const prev = live.get(urlHash);
-							live.delete(urlHash);
-							if (prev) removed.set(urlHash, {
-								...prev,
-								removedAt: at
-							});
-						}
-					}
-				}
-				const merged = [...live.values(), ...removed.values()];
-				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-					columns: URLS_INDEX_COLUMNS,
-					sortKey: ["feedpath_hash", "url_hash"]
-				});
-				await ds.write(indexKey, bytes);
-				if (consumed.length > 0) await ds.delete(consumed);
-			}
-		}
-	};
-}
-function indexingMetadataIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
-}
-function createIndexingMetadataStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readIndex(key) {
-		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
-			version: 1,
-			records: {}
-		}));
-	}
-	return {
-		async writeBatch(ctx, records) {
-			if (records.length === 0) return;
-			const key = indexingMetadataIndexKey(ctx);
-			const index = await readIndex(key);
-			for (const r of records) index.records[hash(r.url)] = r;
-			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
-		},
-		async loadIndex(ctx) {
-			return readIndex(indexingMetadataIndexKey(ctx));
-		},
-		async getLatest(ctx, url) {
-			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
-		}
-	};
-}
-function createEmptyTypesStore(opts) {
-	const ds = opts.dataSource;
-	const now = opts.now ?? (() => Date.now());
-	async function readDoc(key) {
-		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
-			version: 1,
-			emptyTypes: [],
-			markedAt: {}
-		}));
-	}
-	async function writeDoc(key, doc) {
-		await ds.write(key, new TextEncoder().encode(JSON.stringify(doc)));
-	}
-	return {
-		async load(ctx) {
-			return readDoc(emptyTypesKey(ctx));
-		},
-		async mark(ctx, types, at) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const stamp = at ?? now();
-			let changed = false;
-			for (const t of types) {
-				if (!doc.emptyTypes.includes(t)) {
-					doc.emptyTypes.push(t);
-					changed = true;
-				}
-				if (doc.markedAt[t] === void 0) {
-					doc.markedAt[t] = stamp;
-					changed = true;
-				}
-			}
-			if (changed) {
-				doc.emptyTypes.sort();
-				await writeDoc(key, doc);
-			}
-			return doc;
-		},
-		async clear(ctx, types) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const drop = new Set(types);
-			const before = doc.emptyTypes.length;
-			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
-			for (const t of drop) delete doc.markedAt[t];
-			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
-			return doc;
-		}
-	};
-}
+import { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
 export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/errors.d.mts

@@ -0,0 +1,2 @@
+import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError } from "./_chunks/errors.mjs";
+export { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError };