@gscdump/engine 0.17.4 → 0.18.0

package/dist/_chunks/compiler.mjs

@@ -1,5 +1,5 @@
 import { i as dimensionToColumn, r as currentSchemaVersion } from "./schema.mjs";
-import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, s as objectKey } from "./storage.mjs";
+import { a as inferSearchType, c as objectKey, f as weekPartition, l as quarterOfMonth, n as dayPartition, o as mondayOfWeek, s as monthPartition, u as quarterPartition } from "./storage.mjs";
 import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "../sql-fragments.mjs";
 import { MS_PER_DAY } from "gscdump";
 import { buildLogicalPlan } from "gscdump/query/plan";
@@ -75,6 +75,7 @@ async function runStage(deps, ctx, stage, now) {
 	});
 	const buckets = /* @__PURE__ */ new Map();
 	for (const entry of candidates) {
+		if (entry.partition.startsWith("hourly/")) continue;
 		const key = stage.bucketKey(entry);
 		if (!key) continue;
 		if (stage.bucketLatestMs(key) >= cutoff) continue;

package/dist/_chunks/duckdb.d.mts

@@ -1,4 +1,4 @@
-import { S as Row, b as QueryExecutor, j as TableName, p as ParquetCodec } from "./storage.mjs";
+import { C as Row, M as TableName, m as ParquetCodec, x as QueryExecutor } from "./storage.mjs";
 interface DuckDBHandle {
   query: (sql: string, params?: unknown[]) => Promise<Row[]>;
   registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;

package/dist/_chunks/engine.mjs

@@ -1,5 +1,5 @@
 import { r as currentSchemaVersion, t as SCHEMAS } from "./schema.mjs";
-import { i as inferSearchType, n as dayPartition, s as objectKey, u as tenantPrefix } from "./storage.mjs";
+import { a as inferSearchType, c as objectKey, d as tenantPrefix, n as dayPartition, r as hourPartition } from "./storage.mjs";
 import { i as substituteNamedFiles, n as compileLogicalQueryPlan, o as compactTieredImpl } from "./compiler.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
@@ -168,6 +168,7 @@ function defaultForType(t) {
 function columnsJson(table) {
 	return `{${SCHEMAS[table].columns.map((c) => `'${c.name}': '${c.type}'`).join(", ")}}`;
 }
+const DEFAULT_HOURLY_RETENTION_MS = 2160 * 60 * 60 * 1e3;
 const VERSION_RE = /__v(\d+)\.parquet$/;
 function parseLockScope(key) {
 	const match = VERSION_RE.exec(key);
@@ -193,6 +194,19 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 		await deps.dataSource.delete(retired.map((e) => e.objectKey));
 		await deps.manifestStore.delete(retired);
 	}
+	let hourlyDeleted = 0;
+	if (opts.userId) {
+		const hourlyCutoff = now - (opts.hourlyRetentionMs ?? DEFAULT_HOURLY_RETENTION_MS);
+		const expiredHourly = (await deps.manifestStore.listAll({
+			userId: opts.userId,
+			siteId: opts.siteId
+		})).filter((e) => e.partition.startsWith("hourly/") && e.createdAt < hourlyCutoff);
+		if (expiredHourly.length > 0) {
+			await deps.dataSource.delete(expiredHourly.map((e) => e.objectKey));
+			await deps.manifestStore.delete(expiredHourly);
+			hourlyDeleted = expiredHourly.length;
+		}
+	}
 	let sweptOrphans = 0;
 	if (opts.userId) {
 		const prefix = tenantPrefix({
@@ -242,7 +256,7 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 			}
 		});
 	}
-	return { deleted: retired.length + sweptOrphans };
+	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
 }
 const URL_PURGE_TABLES = ["pages", "page_keywords"];
 const MAX_DAY_BYTES = 100 * 1024 * 1024;
@@ -314,6 +328,73 @@ function createStorageEngine(opts) {
 			}, date, now);
 		});
 	}
+	async function writeHour(ctx, rows) {
+		if (!ctx.date) throw new Error("writeHour requires ctx.date (the PT calendar day)");
+		const date = ctx.date;
+		const now = (ctx.now ?? defaultNow)();
+		const partition = hourPartition(date);
+		const searchType = ctx.searchType;
+		return manifestStore.withLock({
+			userId: ctx.userId,
+			siteId: ctx.siteId,
+			table: ctx.table,
+			partition
+		}, async () => {
+			const live = await manifestStore.listLive({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partitions: [partition],
+				searchType: inferSearchType({ searchType })
+			});
+			const existing = [];
+			for (const entry of live) {
+				const rs = await codec.readRows({ table: ctx.table }, entry.objectKey, dataSource);
+				existing.push(...rs);
+			}
+			const dedup = /* @__PURE__ */ new Map();
+			for (const r of existing) {
+				const k = `${String(r.url ?? "")}\0${String(r.hour ?? "")}`;
+				dedup.set(k, r);
+			}
+			for (const r of rows) {
+				const normalized = normalizeRow(ctx.table, r);
+				const k = `${String(normalized.url ?? "")}\0${String(normalized.hour ?? "")}`;
+				dedup.set(k, normalized);
+			}
+			const merged = [...dedup.values()];
+			const key = objectKey(ctx, ctx.table, partition, now, searchType);
+			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, merged, key, dataSource);
+			let bytes = writtenBytes;
+			if (bytes === 0 && rowCount > 0 && dataSource.head) {
+				const probed = await dataSource.head(key);
+				if (probed) bytes = probed.bytes;
+			}
+			if (bytes > 104857600) {
+				await dataSource.delete([key]).catch(() => {});
+				throw new Error(`writeHour payload ${bytes} bytes exceeds ${MAX_DAY_BYTES} hard ceiling (table=${ctx.table}, key=${key})`);
+			}
+			const entry = {
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table,
+				partition,
+				objectKey: key,
+				rowCount,
+				bytes,
+				createdAt: now,
+				schemaVersion: currentSchemaVersion(ctx.table),
+				tier: "raw",
+				...searchType !== void 0 ? { searchType } : {}
+			};
+			await manifestStore.registerVersion(entry, live);
+			await manifestStore.bumpWatermark({
+				userId: ctx.userId,
+				siteId: ctx.siteId,
+				table: ctx.table
+			}, date, now);
+		});
+	}
 	async function runSQL(opts) {
 		opts.signal?.throwIfAborted();
 		const entries = Object.entries(opts.fileSets);
@@ -387,7 +468,8 @@ function createStorageEngine(opts) {
 			manifestStore
 		}, (ctx.now ?? defaultNow)(), graceMs, {
 			userId: ctx.userId,
-			siteId: ctx.siteId
+			siteId: ctx.siteId,
+			...ctx.hourlyRetentionMs !== void 0 ? { hourlyRetentionMs: ctx.hourlyRetentionMs } : {}
 		});
 	}
 	async function purgeTenant(ctx) {
@@ -481,6 +563,7 @@ function createStorageEngine(opts) {
 	}
 	return {
 		writeDay,
+		writeHour,
 		query,
 		runSQL,
 		compactTiered,
@@ -495,4 +578,4 @@ function createStorageEngine(opts) {
 		readObject: (key) => dataSource.read(key)
 	};
 }
-export { createDuckDBExecutor as a, createDuckDBCodec as i, createStorageEngine as n, canonicalEmptyParquetSchema as r, MAX_DAY_BYTES as t };
+export { createDuckDBCodec as a, canonicalEmptyParquetSchema as i, createStorageEngine as n, createDuckDBExecutor as o, gcOrphansImpl as r, MAX_DAY_BYTES as t };

package/dist/_chunks/planner.d.mts

@@ -1,4 +1,4 @@
-import { j as TableName } from "./storage.mjs";
+import { M as TableName } from "./storage.mjs";
 import { LogicalQueryPlan } from "gscdump/query/plan";
 import { BuilderState } from "gscdump/query";
 interface ResolvedQuery {

package/dist/_chunks/registry.d.mts

@@ -1,4 +1,4 @@
-import { S as Row$1 } from "./storage.mjs";
+import { C as Row$1 } from "./storage.mjs";
 import { t as AnalysisParams } from "./analysis-types.mjs";
 import { o as ResolverAdapter } from "./types.mjs";
 import { PlannerCapabilities } from "gscdump/query/plan";