@gscdump/engine 1.4.11 → 1.5.1

package/dist/entities/sitemap-shared.mjs

@@ -0,0 +1,243 @@
+import { readOptional } from "../adapters/read-optional.mjs";
+import { decodeParquetToRows } from "../adapters/hyparquet.mjs";
+import { mapEntityIo } from "./io.mjs";
+const SITEMAP_URLS_EVENT_PREFIX_RE = /\/urls\/events\/(\d{4}-\d{2}-\d{2})__[0-9a-f]+__\d+__[0-9a-f]+\.parquet$/;
+const SITEMAP_URLS_PENDING_GENERATION_RE = /\/urls\/generations\/pending\/([0-9a-f]+)\.json$/;
+const URLS_INDEX_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "removed_at",
+		type: "BIGINT",
+		nullable: true
+	}
+];
+const URLS_DELTA_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "generation_id",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+const URLS_EVENT_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "generation_id",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "observed_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "sequence",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "projects_state",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "seed_generation",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "generation_kind",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "content_hash",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "input_count",
+		type: "INTEGER",
+		nullable: true
+	}
+];
+function rowToUrlRecord(row) {
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		firstSeenAt: Number(row.first_seen_at),
+		lastSeenAt: Number(row.last_seen_at),
+		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
+	};
+}
+function urlRecordToRow(r) {
+	return {
+		feedpath: r.feedpath,
+		feedpath_hash: r.feedpathHash,
+		url_hash: r.urlHash,
+		loc: r.loc,
+		lastmod: r.lastmod ?? null,
+		first_seen_at: r.firstSeenAt,
+		last_seen_at: r.lastSeenAt,
+		removed_at: r.removedAt ?? null
+	};
+}
+function createSitemapUrlState(indexRows) {
+	const state = {
+		live: /* @__PURE__ */ new Map(),
+		removed: /* @__PURE__ */ new Map()
+	};
+	for (const row of indexRows) {
+		const record = rowToUrlRecord(row);
+		if (record.removedAt == null) state.live.set(record.urlHash, record);
+		else state.removed.set(record.urlHash, record);
+	}
+	return state;
+}
+function applySitemapDeltaRows(state, rows) {
+	for (const row of rows) {
+		const urlHash = String(row.url_hash);
+		const observedAt = Number(row.at);
+		const op = String(row.op);
+		if (op === "added") {
+			const previous = state.live.get(urlHash) ?? state.removed.get(urlHash);
+			state.removed.delete(urlHash);
+			state.live.set(urlHash, {
+				feedpath: String(row.feedpath),
+				feedpathHash: String(row.feedpath_hash),
+				urlHash,
+				loc: String(row.loc),
+				lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+				firstSeenAt: previous?.firstSeenAt ?? observedAt,
+				lastSeenAt: observedAt
+			});
+		} else if (op === "removed") {
+			const previous = state.live.get(urlHash);
+			state.live.delete(urlHash);
+			if (previous) state.removed.set(urlHash, {
+				...previous,
+				removedAt: observedAt
+			});
+		}
+	}
+}
+function applySitemapDeltaFiles(state, files) {
+	for (const file of files) applySitemapDeltaRows(state, file.rows);
+}
+function sortSitemapDeltaFiles(files) {
+	return files.sort((a, b) => a.key.localeCompare(b.key));
+}
+async function readSitemapDeltaFiles(ds, keys) {
+	return sortSitemapDeltaFiles((await mapEntityIo(keys, async (key) => {
+		const bytes = await readOptional(ds, key);
+		return bytes ? {
+			key,
+			rows: await decodeParquetToRows(bytes)
+		} : void 0;
+	})).filter((file) => file !== void 0));
+}
+function dateInRange(date, range) {
+	return (!range?.from || date >= range.from) && (!range?.to || date <= range.to);
+}
+export { SITEMAP_URLS_EVENT_PREFIX_RE, SITEMAP_URLS_PENDING_GENERATION_RE, URLS_DELTA_COLUMNS, URLS_EVENT_COLUMNS, URLS_INDEX_COLUMNS, applySitemapDeltaFiles, applySitemapDeltaRows, createSitemapUrlState, dateInRange, readSitemapDeltaFiles, rowToUrlRecord, sortSitemapDeltaFiles, urlRecordToRow };

package/dist/entities/sitemap-write.d.mts

@@ -0,0 +1,3 @@
+import { CreateSitemapStoreOptions, SitemapStore } from "./sitemap-shared.mjs";
+declare function createSitemapStore(opts: CreateSitemapStoreOptions): SitemapStore;
+export { createSitemapStore };