@gscdump/engine 0.25.0 → 0.25.1

package/dist/_chunks/compaction.mjs

@@ -0,0 +1,247 @@
+import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
+import { currentSchemaVersion } from "./schema.mjs";
+import { MS_PER_DAY } from "gscdump";
+const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
+const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
+const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
+const QUARTERLY_PARTITION_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
+const DEFAULT_THRESHOLDS = {
+	raw: 7,
+	d7: 30,
+	d30: 90
+};
+function countRawDailies(entries) {
+	return entries.filter((e) => e.tier === "raw" || e.tier == null && e.partition.startsWith("daily/")).length;
+}
+const PENDING_WINDOW_DAYS = 4;
+const STAGES = [
+	{
+		inputTier: "raw",
+		outputTier: "d7",
+		cutoffDays: DEFAULT_THRESHOLDS.raw,
+		bucketKey: (e) => {
+			const m = e.partition.match(DAILY_PARTITION_RE);
+			if (!m) return void 0;
+			return mondayOfWeek(m[1]);
+		},
+		bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
+		outputPartition: weekPartition
+	},
+	{
+		inputTier: "d7",
+		outputTier: "d30",
+		cutoffDays: DEFAULT_THRESHOLDS.d7,
+		bucketKey: (e) => {
+			const m = e.partition.match(WEEKLY_PARTITION_RE);
+			if (!m) return void 0;
+			return m[1].slice(0, 7);
+		},
+		bucketLatestMs: monthEndMs,
+		outputPartition: monthPartition
+	},
+	{
+		inputTier: "d30",
+		outputTier: "d90",
+		cutoffDays: DEFAULT_THRESHOLDS.d30,
+		bucketKey: (e) => {
+			const m = e.partition.match(MONTHLY_PARTITION_RE);
+			if (!m) return void 0;
+			return quarterOfMonth(m[1]);
+		},
+		bucketLatestMs: quarterEndMs,
+		outputPartition: quarterPartition
+	}
+];
+async function compactTieredImpl(deps, ctx, now, overrides = {}) {
+	const thresholds = {
+		...DEFAULT_THRESHOLDS,
+		...overrides
+	};
+	const stagesWithThresholds = STAGES.map((s) => ({
+		...s,
+		cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
+	}));
+	for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
+}
+async function runStage(deps, ctx, stage, now) {
+	const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
+	const candidates = await deps.manifestStore.listLive({
+		userId: ctx.userId,
+		siteId: ctx.siteId,
+		table: ctx.table,
+		tier: stage.inputTier
+	});
+	const buckets = /* @__PURE__ */ new Map();
+	for (const entry of candidates) {
+		if (entry.partition.startsWith("hourly/")) continue;
+		const key = stage.bucketKey(entry);
+		if (!key) continue;
+		if (stage.bucketLatestMs(key) >= cutoff) continue;
+		const compositeKey = `${inferSearchType(entry)}\0${key}`;
+		if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
+		buckets.get(compositeKey).push(entry);
+	}
+	for (const [compositeKey, entries] of buckets) {
+		const [searchType, bucket] = compositeKey.split("\0");
+		const targetPartition = stage.outputPartition(bucket);
+		if (entries.length === 1 && entries[0].partition === targetPartition) continue;
+		await deps.manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition: targetPartition
+		}, async () => {
+			const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
+			const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
+			const newEntry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition: targetPartition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: stage.outputTier,
+				...searchType !== "web" ? { searchType } : {}
+			};
+			await deps.manifestStore.registerVersion(newEntry, entries);
+		});
+	}
+}
+function enumeratePartitions(startDate, endDate) {
+	const out = [];
+	const [sy, sm, sd] = startDate.split("-").map(Number);
+	const [ey, em, ed] = endDate.split("-").map(Number);
+	const start = Date.UTC(sy, sm - 1, sd);
+	const end = Date.UTC(ey, em - 1, ed);
+	if (end < start) return out;
+	const seenWeeks = /* @__PURE__ */ new Set();
+	const seenMonths = /* @__PURE__ */ new Set();
+	const seenQuarters = /* @__PURE__ */ new Set();
+	for (let t = start; t <= end; t += 864e5) {
+		const d = new Date(t);
+		const y = d.getUTCFullYear();
+		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
+		const isoMonth = `${y}-${m}`;
+		out.push(dayPartition(isoDay));
+		const monday = mondayOfWeek(isoDay);
+		if (!seenWeeks.has(monday)) {
+			seenWeeks.add(monday);
+			out.push(weekPartition(monday));
+		}
+		if (!seenMonths.has(isoMonth)) {
+			seenMonths.add(isoMonth);
+			out.push(monthPartition(isoMonth));
+		}
+		const quarter = quarterOfMonth(isoMonth);
+		if (!seenQuarters.has(quarter)) {
+			seenQuarters.add(quarter);
+			out.push(quarterPartition(quarter));
+		}
+	}
+	return out;
+}
+function partitionSpan(partition) {
+	let m = partition.match(DAILY_PARTITION_RE);
+	if (m) {
+		const ms = Date.parse(`${m[1]}T00:00:00Z`);
+		return {
+			rank: 0,
+			startMs: ms,
+			endMs: ms
+		};
+	}
+	m = partition.match(WEEKLY_PARTITION_RE);
+	if (m) {
+		const ms = Date.parse(`${m[1]}T00:00:00Z`);
+		return {
+			rank: 1,
+			startMs: ms,
+			endMs: ms + 6 * MS_PER_DAY
+		};
+	}
+	m = partition.match(MONTHLY_PARTITION_RE);
+	if (m) {
+		const [y, mo] = m[1].split("-").map(Number);
+		return {
+			rank: 2,
+			startMs: Date.UTC(y, mo - 1, 1),
+			endMs: Date.UTC(y, mo, 0)
+		};
+	}
+	m = partition.match(QUARTERLY_PARTITION_RE);
+	if (m) {
+		const y = Number(m[1]);
+		const q = Number(m[2]);
+		return {
+			rank: 3,
+			startMs: Date.UTC(y, (q - 1) * 3, 1),
+			endMs: Date.UTC(y, q * 3, 0)
+		};
+	}
+}
+function splitOverlappingTiers(entries, queryRange) {
+	const rangeStartMs = queryRange ? Date.parse(`${queryRange.start}T00:00:00Z`) : void 0;
+	const rangeEndMs = queryRange ? Date.parse(`${queryRange.end}T00:00:00Z`) : void 0;
+	const spanned = [];
+	const kept = [];
+	const subsumed = [];
+	for (const entry of entries) {
+		const span = partitionSpan(entry.partition);
+		if (!span) {
+			kept.push(entry);
+			continue;
+		}
+		const days = [];
+		for (let t = span.startMs; t <= span.endMs; t += MS_PER_DAY) {
+			if (rangeStartMs !== void 0 && (t < rangeStartMs || t > rangeEndMs)) continue;
+			days.push(t);
+		}
+		if (queryRange && days.length === 0) {
+			subsumed.push(entry);
+			continue;
+		}
+		spanned.push({
+			entry,
+			rank: span.rank,
+			days
+		});
+	}
+	spanned.sort((a, b) => a.rank - b.rank || b.entry.createdAt - a.entry.createdAt);
+	const coveredBySearchType = /* @__PURE__ */ new Map();
+	for (const { entry, days } of spanned) {
+		const slice = inferSearchType(entry);
+		let covered = coveredBySearchType.get(slice);
+		if (!covered) {
+			covered = /* @__PURE__ */ new Set();
+			coveredBySearchType.set(slice, covered);
+		}
+		if (days.every((d) => covered.has(d))) {
+			subsumed.push(entry);
+			continue;
+		}
+		kept.push(entry);
+		for (const d of days) covered.add(d);
+	}
+	return {
+		kept,
+		subsumed
+	};
+}
+function dedupeOverlappingTiers(entries, queryRange) {
+	return splitOverlappingTiers(entries, queryRange).kept;
+}
+function monthEndMs(month) {
+	const [y, m] = month.split("-").map(Number);
+	return Date.UTC(y, m, 0, 23, 59, 59, 999);
+}
+function quarterEndMs(quarter) {
+	const [yStr, qStr] = quarter.split("-Q");
+	const y = Number(yStr);
+	const q = Number(qStr);
+	return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
+}
+export { compactTieredImpl, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };

package/dist/_chunks/engine.mjs

@@ -1,6 +1,7 @@
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
 import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
-import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
+import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
+import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump";
@@ -485,6 +486,22 @@ function createStorageEngine(opts) {
 			codec
 		}, ctx, (ctx.now ?? defaultNow)(), thresholds);
 	}
+	async function reconcileSubsumed(ctx) {
+		const { subsumed } = splitOverlappingTiers(await manifestStore.listLive({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table
+		}));
+		if (subsumed.length === 0) return {
+			retired: 0,
+			partitions: []
+		};
+		await manifestStore.registerVersions([], subsumed);
+		return {
+			retired: subsumed.length,
+			partitions: subsumed.map((e) => e.partition)
+		};
+	}
 	async function gcOrphans(ctx, graceMs) {
 		return gcOrphansImpl({
 			dataSource,
@@ -590,6 +607,7 @@ function createStorageEngine(opts) {
 		query,
 		runSQL,
 		compactTiered,
+		reconcileSubsumed,
 		gcOrphans,
 		purgeTenant,
 		purgeUrls,

package/dist/_chunks/parquet-plan.mjs

@@ -1,248 +1,7 @@
-import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
-import { currentSchemaVersion, dimensionToColumn } from "./schema.mjs";
+import { dimensionToColumn } from "./schema.mjs";
+import { enumeratePartitions } from "./compaction.mjs";
 import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "../sql-fragments.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
-import { MS_PER_DAY } from "gscdump";
-const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
-const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
-const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
-const QUARTERLY_PARTITION_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
-const DEFAULT_THRESHOLDS = {
-	raw: 7,
-	d7: 30,
-	d30: 90
-};
-const PENDING_WINDOW_DAYS = 4;
-const STAGES = [
-	{
-		inputTier: "raw",
-		outputTier: "d7",
-		cutoffDays: DEFAULT_THRESHOLDS.raw,
-		bucketKey: (e) => {
-			const m = e.partition.match(DAILY_PARTITION_RE);
-			if (!m) return void 0;
-			return mondayOfWeek(m[1]);
-		},
-		bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
-		outputPartition: weekPartition
-	},
-	{
-		inputTier: "d7",
-		outputTier: "d30",
-		cutoffDays: DEFAULT_THRESHOLDS.d7,
-		bucketKey: (e) => {
-			const m = e.partition.match(WEEKLY_PARTITION_RE);
-			if (!m) return void 0;
-			return m[1].slice(0, 7);
-		},
-		bucketLatestMs: monthEndMs,
-		outputPartition: monthPartition
-	},
-	{
-		inputTier: "d30",
-		outputTier: "d90",
-		cutoffDays: DEFAULT_THRESHOLDS.d30,
-		bucketKey: (e) => {
-			const m = e.partition.match(MONTHLY_PARTITION_RE);
-			if (!m) return void 0;
-			return quarterOfMonth(m[1]);
-		},
-		bucketLatestMs: quarterEndMs,
-		outputPartition: quarterPartition
-	}
-];
-async function compactTieredImpl(deps, ctx, now, overrides = {}) {
-	const thresholds = {
-		...DEFAULT_THRESHOLDS,
-		...overrides
-	};
-	const stagesWithThresholds = STAGES.map((s) => ({
-		...s,
-		cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
-	}));
-	for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
-}
-async function runStage(deps, ctx, stage, now) {
-	const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
-	const candidates = await deps.manifestStore.listLive({
-		userId: ctx.userId,
-		siteId: ctx.siteId,
-		table: ctx.table,
-		tier: stage.inputTier
-	});
-	const buckets = /* @__PURE__ */ new Map();
-	for (const entry of candidates) {
-		if (entry.partition.startsWith("hourly/")) continue;
-		const key = stage.bucketKey(entry);
-		if (!key) continue;
-		if (stage.bucketLatestMs(key) >= cutoff) continue;
-		const compositeKey = `${inferSearchType(entry)}\0${key}`;
-		if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
-		buckets.get(compositeKey).push(entry);
-	}
-	for (const [compositeKey, entries] of buckets) {
-		const [searchType, bucket] = compositeKey.split("\0");
-		const targetPartition = stage.outputPartition(bucket);
-		if (entries.length === 1 && entries[0].partition === targetPartition) continue;
-		await deps.manifestStore.withLock({
-			userId: ctx.userId,
-			siteId: ctx.siteId,
-			table: ctx.table,
-			partition: targetPartition
-		}, async () => {
-			const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
-			const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
-			const newEntry = {
-				userId: ctx.userId,
-				siteId: ctx.siteId,
-				table: ctx.table,
-				partition: targetPartition,
-				objectKey: key,
-				rowCount,
-				bytes,
-				createdAt: now,
-				schemaVersion: currentSchemaVersion(ctx.table),
-				tier: stage.outputTier,
-				...searchType !== "web" ? { searchType } : {}
-			};
-			await deps.manifestStore.registerVersion(newEntry, entries);
-		});
-	}
-}
-function enumeratePartitions(startDate, endDate) {
-	const out = [];
-	const [sy, sm, sd] = startDate.split("-").map(Number);
-	const [ey, em, ed] = endDate.split("-").map(Number);
-	const start = Date.UTC(sy, sm - 1, sd);
-	const end = Date.UTC(ey, em - 1, ed);
-	if (end < start) return out;
-	const seenWeeks = /* @__PURE__ */ new Set();
-	const seenMonths = /* @__PURE__ */ new Set();
-	const seenQuarters = /* @__PURE__ */ new Set();
-	for (let t = start; t <= end; t += 864e5) {
-		const d = new Date(t);
-		const y = d.getUTCFullYear();
-		const m = String(d.getUTCMonth() + 1).padStart(2, "0");
-		const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
-		const isoMonth = `${y}-${m}`;
-		out.push(dayPartition(isoDay));
-		const monday = mondayOfWeek(isoDay);
-		if (!seenWeeks.has(monday)) {
-			seenWeeks.add(monday);
-			out.push(weekPartition(monday));
-		}
-		if (!seenMonths.has(isoMonth)) {
-			seenMonths.add(isoMonth);
-			out.push(monthPartition(isoMonth));
-		}
-		const quarter = quarterOfMonth(isoMonth);
-		if (!seenQuarters.has(quarter)) {
-			seenQuarters.add(quarter);
-			out.push(quarterPartition(quarter));
-		}
-	}
-	return out;
-}
-function partitionSpan(partition) {
-	let m = partition.match(DAILY_PARTITION_RE);
-	if (m) {
-		const ms = Date.parse(`${m[1]}T00:00:00Z`);
-		return {
-			rank: 0,
-			startMs: ms,
-			endMs: ms
-		};
-	}
-	m = partition.match(WEEKLY_PARTITION_RE);
-	if (m) {
-		const ms = Date.parse(`${m[1]}T00:00:00Z`);
-		return {
-			rank: 1,
-			startMs: ms,
-			endMs: ms + 6 * MS_PER_DAY
-		};
-	}
-	m = partition.match(MONTHLY_PARTITION_RE);
-	if (m) {
-		const [y, mo] = m[1].split("-").map(Number);
-		return {
-			rank: 2,
-			startMs: Date.UTC(y, mo - 1, 1),
-			endMs: Date.UTC(y, mo, 0)
-		};
-	}
-	m = partition.match(QUARTERLY_PARTITION_RE);
-	if (m) {
-		const y = Number(m[1]);
-		const q = Number(m[2]);
-		return {
-			rank: 3,
-			startMs: Date.UTC(y, (q - 1) * 3, 1),
-			endMs: Date.UTC(y, q * 3, 0)
-		};
-	}
-}
-function splitOverlappingTiers(entries, queryRange) {
-	const rangeStartMs = queryRange ? Date.parse(`${queryRange.start}T00:00:00Z`) : void 0;
-	const rangeEndMs = queryRange ? Date.parse(`${queryRange.end}T00:00:00Z`) : void 0;
-	const spanned = [];
-	const kept = [];
-	const subsumed = [];
-	for (const entry of entries) {
-		const span = partitionSpan(entry.partition);
-		if (!span) {
-			kept.push(entry);
-			continue;
-		}
-		const days = [];
-		for (let t = span.startMs; t <= span.endMs; t += MS_PER_DAY) {
-			if (rangeStartMs !== void 0 && (t < rangeStartMs || t > rangeEndMs)) continue;
-			days.push(t);
-		}
-		if (queryRange && days.length === 0) {
-			subsumed.push(entry);
-			continue;
-		}
-		spanned.push({
-			entry,
-			rank: span.rank,
-			days
-		});
-	}
-	spanned.sort((a, b) => a.rank - b.rank || b.entry.createdAt - a.entry.createdAt);
-	const coveredBySearchType = /* @__PURE__ */ new Map();
-	for (const { entry, days } of spanned) {
-		const slice = inferSearchType(entry);
-		let covered = coveredBySearchType.get(slice);
-		if (!covered) {
-			covered = /* @__PURE__ */ new Set();
-			coveredBySearchType.set(slice, covered);
-		}
-		if (days.every((d) => covered.has(d))) {
-			subsumed.push(entry);
-			continue;
-		}
-		kept.push(entry);
-		for (const d of days) covered.add(d);
-	}
-	return {
-		kept,
-		subsumed
-	};
-}
-function dedupeOverlappingTiers(entries, queryRange) {
-	return splitOverlappingTiers(entries, queryRange).kept;
-}
-function monthEndMs(month) {
-	const [y, m] = month.split("-").map(Number);
-	return Date.UTC(y, m, 0, 23, 59, 59, 999);
-}
-function quarterEndMs(quarter) {
-	const [yStr, qStr] = quarter.split("-Q");
-	const y = Number(yStr);
-	const q = Number(qStr);
-	return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
-}
 const FILES_PLACEHOLDER = "{{FILES}}";
 function buildDimensionWhere(filters, table) {
 	const clauses = [];
@@ -377,4 +136,4 @@ function substituteNamedFiles(sql, sets) {
 	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
 	return out;
 }
-export { FILES_PLACEHOLDER, compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };
+export { FILES_PLACEHOLDER, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles };

package/dist/_chunks/resolver.mjs

@@ -1,5 +1,5 @@
 import { SCHEMAS, drizzleSchema } from "./schema.mjs";
-import { enumeratePartitions } from "./parquet-plan.mjs";
+import { enumeratePartitions } from "./compaction.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";

package/dist/_chunks/storage.d.mts

@@ -12,6 +12,48 @@ interface CompactionThresholds {
   d30?: number;
 }
 declare function enumeratePartitions(startDate: string, endDate: string): string[];
+/**
+ * Split manifest entries into the set worth reading (`kept`) and the set whose
+ * every covered day is already served by a finer-or-newer live entry
+ * (`subsumed`).
+ *
+ * Tiered compaction (daily→weekly→monthly→quarterly) is meant to retire its
+ * inputs, but coarse files can outlive their finer counterparts: a D1→R2
+ * backfill writes daily files that compact to monthly while a later re-sync
+ * writes fresh daily/weekly for the same dates, and same-partition re-writes
+ * leave a stale prior version live. All stay live, the resolver unions every
+ * live tier whose partition intersects the range, and `union_by_name` sums the
+ * overlap — impressions/clicks double-count.
+ *
+ * Entries are walked finest-tier-first, newest-first within a tier, so a
+ * coarse or stale file is dropped only when every day it covers is already
+ * claimed. Subsumption is evaluated per searchType — a `web` monthly never
+ * cancels a `discover` weekly, they cover disjoint data. Partial
+ * month-boundary overlap (a weekly straddling two months alongside a kept
+ * monthly) still double-counts those boundary days — eliminating that needs
+ * per-file date predicates in the SQL, tracked separately. Unrecognised
+ * partition shapes (`hourly/`, sidecar keys) are always kept.
+ *
+ * `queryRange` clamps every entry's day-span to the window the caller will
+ * actually read. This is required when `entries` came from a partition-
+ * filtered `listLive` (`runSQL` enumerates only the partitions intersecting
+ * the query): a `monthly/2026-04` whose Apr 27-30 falls past the query end
+ * must not be judged "unsubsumed" just because `weekly/2026-04-27` wasn't
+ * enumerated — those out-of-window days are SQL-filtered to nothing anyway.
+ * Omit `queryRange` when `entries` is the full manifest (e.g. analysis-sources).
+ */
+declare function splitOverlappingTiers(entries: ManifestEntry[], queryRange?: {
+  start: string;
+  end: string;
+}): {
+  kept: ManifestEntry[];
+  subsumed: ManifestEntry[];
+};
+/** Entries worth reading — see {@link splitOverlappingTiers}. */
+declare function dedupeOverlappingTiers(entries: ManifestEntry[], queryRange?: {
+  start: string;
+  end: string;
+}): ManifestEntry[];
 interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;
@@ -428,6 +470,29 @@ interface StorageEngine {
    */
   runSQL: (opts: RunSQLOptions) => Promise<QueryResult>;
   compactTiered: (ctx: WriteCtx, thresholds?: CompactionThresholds) => Promise<void>;
+  /**
+   * Write-time half of the manifest tier invariant: retire every live entry
+   * whose every covered day is already served by a finer-or-newer live entry.
+   *
+   * `compactTiered` retires the inputs it merges, but cannot retire a coarse
+   * partition that outlived the finer files it should have superseded (a D1→R2
+   * backfill writing coarse directly, a re-sync landing fresh dailies after a
+   * month already rolled up). Those stale overlaps make the query resolver
+   * union the same dates twice. Subsumption is evaluated per searchType, over
+   * the full live set (so a `web` monthly never cancels a `discover` weekly),
+   * then the subsumed set is retired via the manifest's `registerVersions([], …)`
+   * primitive — atomic, no inserts. Safe by construction: it only drops files
+   * whose days are already covered, so no data is lost.
+   *
+   * Reads and retires through the engine's own manifest store, so it is
+   * read-your-writes-consistent with the `compactTiered` that precedes it.
+   * Returns audit counters. Hosts running a cached manifest store must bust
+   * their cache afterwards — the engine has no knowledge of host-side caching.
+   */
+  reconcileSubsumed: (ctx: WriteCtx) => Promise<{
+    retired: number;
+    partitions: string[];
+  }>;
   gcOrphans: (ctx: GcCtx, graceMs: number) => Promise<{
     deleted: number;
   }>;
@@ -476,4 +541,4 @@ interface EngineOptions {
   executor: QueryExecutor;
   now?: () => number;
 }
-export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions };
+export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };

package/dist/compaction-public.d.mts

@@ -0,0 +1,15 @@
+import { dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/storage.mjs";
+/**
+ * Host-policy predicate: true once a table's live raw-daily count crosses the
+ * engine's daily→weekly compaction gate. Wraps the internal threshold so hosts
+ * decide "is compaction due?" without importing the constant or the counter.
+ *
+ * Pure — pass the entries the host already fetched (typically via its own cached
+ * manifest store, so the hot-path check stays on the host's cache rather than
+ * forcing an uncached read through the engine).
+ */
+declare function isRawDailyCompactionDue(entries: ReadonlyArray<{
+  tier?: string | null;
+  partition: string;
+}>): boolean;
+export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/compaction-public.mjs

@@ -0,0 +1,5 @@
+import { countRawDailies, dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/compaction.mjs";
+function isRawDailyCompactionDue(entries) {
+	return countRawDailies(entries) > 7;
+}
+export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/index.mjs

@@ -1,7 +1,8 @@
 import { ENGINE_QUERY_CAPABILITIES, coerceRow, coerceRows, createSqlQuerySource } from "./_chunks/source.mjs";
 import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/layout.mjs";
 import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
-import { FILES_PLACEHOLDER, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
+import { enumeratePartitions } from "./_chunks/compaction.mjs";
+import { FILES_PLACEHOLDER, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
 import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";

package/dist/planner.mjs

@@ -1,2 +1,3 @@
-import { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
+import { enumeratePartitions } from "./_chunks/compaction.mjs";
+import { FILES_PLACEHOLDER, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.25.0",
+  "version": "0.25.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -41,6 +41,11 @@
       "import": "./dist/planner.mjs",
       "default": "./dist/planner.mjs"
     },
+    "./compaction": {
+      "types": "./dist/compaction-public.d.mts",
+      "import": "./dist/compaction-public.mjs",
+      "default": "./dist/compaction-public.mjs"
+    },
     "./schema": {
       "types": "./dist/schema.d.mts",
       "import": "./dist/schema.mjs",
@@ -180,8 +185,8 @@
     "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.6",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.25.0",
-    "@gscdump/contracts": "0.25.0"
+    "@gscdump/contracts": "0.25.1",
+    "gscdump": "0.25.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",