@gscdump/engine 1.4.0 → 1.4.1

package/dist/engine.mjs

@@ -0,0 +1,385 @@
+import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
+import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
+import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
+import { compileLogicalQueryPlan } from "./parquet-plan.mjs";
+import { gcOrphansImpl } from "./gc.mjs";
+import { extractParquetPushdown } from "./parquet-pushdown.mjs";
+import { buildLogicalPlan } from "gscdump/query/plan";
+import { normalizeUrl } from "gscdump/normalize";
+const URL_PURGE_TABLES = ["pages", "page_queries"];
+const MAX_DAY_BYTES = 100 * 1024 * 1024;
+const URL_COLUMNS = /* @__PURE__ */ new Set();
+for (const t of Object.keys(SCHEMAS)) for (const col of SCHEMAS[t].columns) if (col.name === "url") URL_COLUMNS.add(`${t}:url`);
+function normalizeRow(table, row) {
+	if (!URL_COLUMNS.has(`${table}:url`)) return row;
+	const url = row.url;
+	if (typeof url !== "string") return row;
+	const normalized = normalizeUrl(url);
+	if (normalized === url) return row;
+	return {
+		...row,
+		url: normalized
+	};
+}
+const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
+function queryRangeOf(partitions) {
+	if (!partitions) return void 0;
+	let min;
+	let max;
+	for (const p of partitions) {
+		const m = DAILY_PARTITION_RE.exec(p);
+		if (!m) continue;
+		const d = m[1];
+		if (min === void 0 || d < min) min = d;
+		if (max === void 0 || d > max) max = d;
+	}
+	return min !== void 0 ? {
+		start: min,
+		end: max
+	} : void 0;
+}
+function createStorageEngine(opts) {
+	const { dataSource, manifestStore, codec, executor } = opts;
+	const defaultNow = opts.now ?? (() => Date.now());
+	async function writeDay(ctx, rows) {
+		if (!ctx.date) throw new Error("writeDay requires ctx.date");
+		const date = ctx.date;
+		const now = (ctx.now ?? defaultNow)();
+		const partition = dayPartition(date);
+		const searchType = ctx.searchType;
+		return manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition
+		}, async () => {
+			const superseding = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partitions: [partition],
+				searchType: inferSearchType({ searchType })
+			});
+			const normalizedRows = dedupeByNaturalKey(ctx.table, rows.map((r) => normalizeRow(ctx.table, r)));
+			const key = objectKey(ctx, ctx.table, partition, now, searchType);
+			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
+			let bytes = writtenBytes;
+			if (bytes === 0 && rowCount > 0 && dataSource.head) {
+				const probed = await dataSource.head(key);
+				if (probed) bytes = probed.bytes;
+			}
+			if (bytes > 104857600) {
+				const sizeError = /* @__PURE__ */ new Error(`writeDay payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				try {
+					await dataSource.delete([key]);
+				} catch (deleteError) {
+					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
+				}
+				throw sizeError;
+			}
+			const entry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: "raw",
+				...searchType !== void 0 ? { searchType } : {}
+			};
+			await manifestStore.registerVersion(entry, superseding);
+			await manifestStore.bumpWatermark({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				...searchType !== void 0 ? { searchType } : {}
+			}, date, now);
+		});
+	}
+	async function writeHour(ctx, rows) {
+		if (!ctx.date) throw new Error("writeHour requires ctx.date (the PT calendar day)");
+		const date = ctx.date;
+		const now = (ctx.now ?? defaultNow)();
+		const partition = hourPartition(date);
+		const searchType = ctx.searchType;
+		return manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition
+		}, async () => {
+			const live = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partitions: [partition],
+				searchType: inferSearchType({ searchType })
+			});
+			const existing = [];
+			for (const entry of live) {
+				const rs = await codec.readRows({ table: ctx.table }, entry.objectKey, dataSource);
+				existing.push(...rs);
+			}
+			const dedup = /* @__PURE__ */ new Map();
+			for (const r of existing) {
+				const k = `${String(r.url ?? "")}\0${String(r.hour ?? "")}`;
+				dedup.set(k, r);
+			}
+			for (const r of rows) {
+				const normalized = normalizeRow(ctx.table, r);
+				const k = `${String(normalized.url ?? "")}\0${String(normalized.hour ?? "")}`;
+				dedup.set(k, normalized);
+			}
+			const merged = [...dedup.values()];
+			const key = objectKey(ctx, ctx.table, partition, now, searchType);
+			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, merged, key, dataSource);
+			let bytes = writtenBytes;
+			if (bytes === 0 && rowCount > 0 && dataSource.head) {
+				const probed = await dataSource.head(key);
+				if (probed) bytes = probed.bytes;
+			}
+			if (bytes > 104857600) {
+				const sizeError = /* @__PURE__ */ new Error(`writeHour payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+				try {
+					await dataSource.delete([key]);
+				} catch (deleteError) {
+					throw new AggregateError([sizeError, deleteError], `${sizeError.message}; failed to delete oversized object`);
+				}
+				throw sizeError;
+			}
+			const entry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: "raw",
+				...searchType !== void 0 ? { searchType } : {}
+			};
+			await manifestStore.registerVersion(entry, live);
+			await manifestStore.bumpWatermark({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				...searchType !== void 0 ? { searchType } : {}
+			}, date, now);
+		});
+	}
+	async function runSQL(opts) {
+		opts.signal?.throwIfAborted();
+		const profiler = opts.profiler;
+		const entries = Object.entries(opts.fileSets);
+		const endList = profiler?.start("manifest.list", { fileSets: entries.length });
+		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
+			if (ref.keys !== void 0) return [name, ref.keys];
+			return [name, dedupeOverlappingTiers(await manifestStore.listLive({
+				userId: opts.ctx.userId,
+				siteId: opts.ctx.siteId,
+				table: ref.table,
+				partitions: ref.partitions,
+				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+			}), queryRangeOf(ref.partitions)).map((e) => e.objectKey)];
+		}));
+		opts.signal?.throwIfAborted();
+		const fileKeys = {};
+		for (const [name, keys] of perSet) fileKeys[name] = keys;
+		const uniqueKeys = [...new Set(perSet.flatMap(([, keys]) => keys))];
+		endList?.({ files: uniqueKeys.length });
+		let table = opts.table;
+		if (!table) {
+			if (new Set(entries.map(([, ref]) => ref.table)).size > 1) throw new Error("runSQL requires explicit ctx.table when fileSets reference multiple tables.");
+			table = entries[0]?.[1].table;
+		}
+		if (!table) throw new Error("runSQL requires at least one fileSet or an explicit table");
+		const placeholderTables = {};
+		for (const [name, ref] of entries) placeholderTables[name] = ref.table;
+		const endExec = profiler?.start("executor.execute", { files: uniqueKeys.length });
+		const result = await executor.execute({
+			sql: opts.sql,
+			params: opts.params ?? [],
+			fileKeys,
+			placeholderTables,
+			dataSource,
+			table,
+			signal: opts.signal,
+			...opts.pushdownFilters ? { pushdownFilters: opts.pushdownFilters } : {},
+			...profiler ? { profiler } : {}
+		});
+		endExec?.({ rows: result.rows.length });
+		return {
+			rows: result.rows,
+			sql: result.sql,
+			objectKeys: uniqueKeys
+		};
+	}
+	async function query(ctx, state) {
+		const plan = buildLogicalPlan(state, { regex: true });
+		const table = ctx.table ?? plan.dataset;
+		const resolved = compileLogicalQueryPlan(plan, table);
+		const pushdown = extractParquetPushdown(state, table);
+		return runSQL({
+			ctx: {
+				userId: ctx.userId,
+				siteId: ctx.siteId
+			},
+			table,
+			fileSets: { FILES: {
+				table,
+				partitions: resolved.partitions
+			} },
+			sql: resolved.sql,
+			params: resolved.params,
+			signal: ctx.signal,
+			...pushdown ? { pushdownFilters: { FILES: pushdown } } : {},
+			...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {},
+			...ctx.profiler ? { profiler: ctx.profiler } : {}
+		});
+	}
+	async function compactTiered(ctx, thresholds) {
+		return compactTieredImpl({
+			dataSource,
+			manifestStore,
+			codec
+		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
+	}
+	async function reconcileSubsumed(ctx) {
+		const { subsumed } = splitOverlappingTiers(await manifestStore.listLive({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table
+		}));
+		if (subsumed.length === 0) return {
+			retired: 0,
+			partitions: []
+		};
+		await manifestStore.registerVersions([], subsumed);
+		return {
+			retired: subsumed.length,
+			partitions: subsumed.map((e) => e.partition)
+		};
+	}
+	async function gcOrphans(ctx, graceMs) {
+		return gcOrphansImpl({
+			dataSource,
+			manifestStore
+		}, (ctx.now ?? defaultNow)(), graceMs, {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			...ctx.hourlyRetentionMs !== void 0 ? { hourlyRetentionMs: ctx.hourlyRetentionMs } : {}
+		});
+	}
+	async function purgeTenant(ctx) {
+		const prefix = tenantPrefix(ctx);
+		const keys = [];
+		const keyStream = dataSource.streamList ? dataSource.streamList(prefix) : async function* () {
+			for (const k of await dataSource.list(prefix)) yield k;
+		}();
+		for await (const key of keyStream) keys.push(key);
+		const manifestResult = await manifestStore.purgeTenant({
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		});
+		if (keys.length > 0) await dataSource.delete(keys);
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			prefix,
+			objectsDeleted: keys.length,
+			entriesRemoved: manifestResult.entriesRemoved,
+			watermarksRemoved: manifestResult.watermarksRemoved,
+			syncStatesRemoved: manifestResult.syncStatesRemoved,
+			at: defaultNow()
+		};
+	}
+	async function purgeUrls(ctx, urls) {
+		const now = defaultNow();
+		const urlSet = new Set(urls);
+		let entriesRewritten = 0;
+		let rowsRemoved = 0;
+		let bytesAfter = 0;
+		if (urlSet.size === 0) return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: 0,
+			entriesRewritten: 0,
+			rowsRemoved: 0,
+			bytesAfter: 0,
+			at: now
+		};
+		for (const table of URL_PURGE_TABLES) {
+			const entries = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table
+			});
+			for (const entry of entries) await manifestStore.withLock({
+				userId: entry.userId,
+				siteId: entry.siteId,
+				table,
+				partition: entry.partition
+			}, async () => {
+				const rows = await codec.readRows({ table }, entry.objectKey, dataSource);
+				const kept = rows.filter((r) => typeof r.url !== "string" || !urlSet.has(r.url));
+				const removed = rows.length - kept.length;
+				if (removed === 0) return;
+				const searchType = entry.searchType;
+				const newKey = objectKey({
+					userId: entry.userId,
+					siteId: entry.siteId
+				}, table, entry.partition, now, searchType);
+				const { bytes, rowCount } = await codec.writeRows({ table }, kept, newKey, dataSource);
+				const newEntry = {
+					userId: entry.userId,
+					siteId: entry.siteId,
+					table,
+					partition: entry.partition,
+					objectKey: newKey,
+					rowCount,
+					bytes,
+					createdAt: now,
+					schemaVersion: entry.schemaVersion ?? currentSchemaVersion(table),
+					...entry.tier !== void 0 ? { tier: entry.tier } : {},
+					...searchType !== void 0 ? { searchType } : {}
+				};
+				await manifestStore.registerVersion(newEntry, [entry]);
+				entriesRewritten++;
+				rowsRemoved += removed;
+				bytesAfter += bytes;
+			});
+		}
+		return {
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			urlsRequested: urlSet.size,
+			entriesRewritten,
+			rowsRemoved,
+			bytesAfter,
+			at: now
+		};
+	}
+	return {
+		writeDay,
+		writeHour,
+		query,
+		runSQL,
+		compactTiered,
+		reconcileSubsumed,
+		gcOrphans,
+		purgeTenant,
+		purgeUrls,
+		listLive: (filter) => manifestStore.listLive(filter),
+		listAll: (filter) => manifestStore.listAll(filter),
+		getWatermarks: (filter) => manifestStore.getWatermarks(filter),
+		getSyncStates: (filter) => manifestStore.getSyncStates(filter),
+		setSyncState: (scope, state, detail) => manifestStore.setSyncState(scope, state, detail),
+		readObject: (key) => dataSource.read(key)
+	};
+}
+export { MAX_DAY_BYTES, createStorageEngine };

package/dist/entities.d.mts

@@ -1,57 +1,8 @@
-import { DataSource } from "./_chunks/storage.mjs";
-import { ScheduleState } from "./_chunks/schedule.mjs";
+import { DataSource } from "./storage.mjs";
+import { ScheduleState } from "./schedule.mjs";
 import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { TenantCtx } from "@gscdump/contracts";
-interface QueryDimRecord {
-  query: string;
-  /** Lexical canonical, never empty: NULL/'' folds to the raw query. */
-  query_canonical: string;
-  /** Packed search-intent code (see `@gscdump/analysis` `encodeIntent`). */
-  intent_code: number;
-  normalizer_version: number;
-  intent_version: number;
-}
-/** JSON sidecar: versions + freshness, readable without decoding the parquet. */
-interface QueryDimMeta {
-  version: 1;
-  builtAt: number;
-  rowCount: number;
-  normalizerVersion: number;
-  intentVersion: number;
-}
-/**
- * Injected derivation. `engine` never imports `@gscdump/analysis`; the host
- * passes `normalizeQuery` / `classifyIntentCode` (e.g. `encodeIntent ∘
- * classifyQueryIntent`) plus their version constants.
- */
-interface QueryDimDeps {
-  normalizeQuery: (query: string) => string;
-  normalizerVersion: number;
-  /** Returns the packed intent code for a raw query. */
-  classifyIntentCode: (query: string) => number;
-  intentVersion: number;
-}
-/**
- * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
- * folds an empty/whitespace canonical back to the raw query so the dimension
- * is total for read-time joins.
- */
-declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
-interface QueryDimStore {
-  parquetKey: (ctx: TenantCtx) => string;
-  /** Write the parquet + JSON sidecar. Last-write-wins; no history. */
-  write: (ctx: TenantCtx, records: readonly QueryDimRecord[], builtAt: number) => Promise<{
-    parquetKey: string;
-    rowCount: number;
-  }>;
-  /** Read the sidecar (versions + freshness), or null on first build. */
-  loadMeta: (ctx: TenantCtx) => Promise<QueryDimMeta | null>;
-  /** Decode the dimension rows (test/inspection; reads JOIN the parquet by key). */
-  loadRecords: (ctx: TenantCtx) => Promise<QueryDimRecord[]>;
-}
-declare function createQueryDimStore({ dataSource }: {
-  dataSource: DataSource;
-}): QueryDimStore;
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON