@gscdump/engine 0.17.5 → 0.18.1

package/dist/_chunks/schema.mjs

@@ -40,13 +40,20 @@ const search_appearance = pgTable("search_appearance", {
 	date: dateCol(),
 	...metricCols()
 });
+const hourly_pages = pgTable("hourly_pages", {
+	url: varchar("url").notNull(),
+	hour: varchar("hour").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
 const drizzleSchema = {
 	pages,
 	keywords,
 	countries,
 	devices,
 	page_keywords,
-	search_appearance
+	search_appearance,
+	hourly_pages
 };
 const TABLE_METADATA = {
 	pages: {
@@ -76,6 +83,14 @@ const TABLE_METADATA = {
 	search_appearance: {
 		sortKey: ["date", "searchAppearance"],
 		version: 1
+	},
+	hourly_pages: {
+		sortKey: [
+			"date",
+			"hour",
+			"url"
+		],
+		version: 1
 	}
 };
 function pgSqlTypeToColumnType(sqlType) {
@@ -107,7 +122,8 @@ const METRIC_TABLES = [
 	"countries",
 	"devices",
 	"page_keywords",
-	"search_appearance"
+	"search_appearance",
+	"hourly_pages"
 ];
 const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
 function currentSchemaVersion(table) {
@@ -136,4 +152,4 @@ function dimensionToColumn(dim, _table) {
 	if (dim === "queryCanonical") return "query_canonical";
 	return dim;
 }
-export { inferTable as a, countries as c, keywords as d, page_keywords as f, dimensionToColumn as i, devices as l, search_appearance as m, allTables as n, schemaFor as o, pages as p, currentSchemaVersion as r, TABLE_METADATA as s, SCHEMAS as t, drizzleSchema as u };
+export { inferTable as a, countries as c, hourly_pages as d, keywords as f, search_appearance as h, dimensionToColumn as i, devices as l, pages as m, allTables as n, schemaFor as o, page_keywords as p, currentSchemaVersion as r, TABLE_METADATA as s, SCHEMAS as t, drizzleSchema as u };

package/dist/_chunks/storage.d.mts

@@ -1,4 +1,4 @@
-import { Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 /**
  * Per-tier age threshold in days. Default ladder collapses on these gates:
@@ -34,6 +34,12 @@ interface WriteCtx extends TenantCtx {
    * for different search types coexist without colliding.
    */
   searchType?: SearchType;
+  /**
+   * Temporal granularity for this write. `'day'` (default) routes to
+   * `writeDay` semantics. `'hour'` routes to `writeHour` — the host (ingest
+   * accumulator) interprets this; the engine surfaces both methods directly.
+   */
+  grain?: Grain;
 }
 interface QueryCtx extends TenantCtx {
   table?: TableName;
@@ -45,11 +51,24 @@ interface QueryCtx extends TenantCtx {
    * manifest entries written for that type. Mirrors {@link WriteCtx.searchType}.
    */
   searchType?: SearchType;
+  /**
+   * Temporal granularity for this query. `'day'` (default) reads daily
+   * partitions only and skips any `hourly/` partitions. `'hour'` reads only
+   * hourly partitions. The two never mix — daily-from-hourly aggregation
+   * happens through the `discover-daily-from-hourly` rollup, not at read.
+   */
+  grain?: Grain;
 }
 interface GcCtx {
   now?: () => number;
   userId?: string;
   siteId?: string;
+  /**
+   * Override retention for hourly partitions in milliseconds. Defaults to
+   * 90 days inside `gcOrphansImpl`. Hourly is GC-only — never compacted —
+   * so this is the only lifecycle knob for `hourly/{date}` entries.
+   */
+  hourlyRetentionMs?: number;
 }
 /**
  * Compaction tier of a manifest entry. Determines which compactor stage may
@@ -414,6 +433,15 @@ interface RunSQLOptions {
 }
 interface StorageEngine {
   writeDay: (ctx: WriteCtx, rows: Row[]) => Promise<void>;
+  /**
+   * Read-merge-write a single-day hourly partition. Idempotent on
+   * `(url, hour)` (last-write-wins): callers can re-fire the same slice
+   * after a retry and the partition converges. `ctx.date` is the PT
+   * calendar day; rows must carry `hour` + `date` fields. Partition shape
+   *  `hourly/{date}`; coexists with daily partitions in the same `table`
+   *  prefix (`hourly_pages`).
+   */
+  writeHour: (ctx: WriteCtx, rows: Row[]) => Promise<void>;
   query: (ctx: QueryCtx, state: BuilderState) => Promise<QueryResult>;
   /**
    * Run arbitrary SQL resolved against named partition sets. Composes
@@ -472,5 +500,12 @@ interface EngineOptions {
   now?: () => number;
 }
 declare function dayPartition(date: string): string;
+/**
+ * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
+ * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
+ * idempotency across retries. Names sort lexically alongside daily ones but
+ * never collide because of the `hourly/` prefix.
+ */
+declare function hourPartition(date: string): string;
 declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
-export { SyncStateScope as A, inferSearchType as B, RunSQLOptions as C, SyncStateDetail as D, SyncState as E, WatermarkScope as F, enumeratePartitions as G, CompactionThresholds as H, WriteCtx as I, WriteResult as L, TenantCtx$1 as M, Watermark as N, SyncStateFilter as O, WatermarkFilter as P, dayPartition as R, Row$1 as S, StorageEngine as T, RAW_DAILY_COMPACT_THRESHOLD as U, objectKey as V, countRawDailies as W, QueryCtx as _, EngineOptions as a, QueryExecutor as b, ListLiveFilter as c, ManifestPurgeResult as d, ManifestStore as f, PurgeUrlsResult as g, PurgeResult as h, DataSource as i, TableName$1 as j, SyncStateKind as k, LockScope as l, PurgeFilter as m, CompactionTier as n, FileSetRef as o, ParquetCodec as p, DEFAULT_SEARCH_TYPE as r, GcCtx as s, CodecCtx as t, ManifestEntry as u, QueryExecuteOptions as v, SearchType$1 as w, QueryResult as x, QueryExecuteResult as y, inferLegacyTier as z };
+export { SyncStateKind as A, hourPartition as B, Row$1 as C, SyncState as D, StorageEngine as E, WatermarkFilter as F, RAW_DAILY_COMPACT_THRESHOLD as G, inferSearchType as H, WatermarkScope as I, countRawDailies as K, WriteCtx as L, TableName$1 as M, TenantCtx$1 as N, SyncStateDetail as O, Watermark as P, WriteResult as R, QueryResult as S, SearchType$1 as T, objectKey as U, inferLegacyTier as V, CompactionThresholds as W, PurgeUrlsResult as _, EngineOptions as a, QueryExecuteResult as b, Grain$1 as c, ManifestEntry as d, ManifestPurgeResult as f, PurgeResult as g, PurgeFilter as h, DataSource as i, SyncStateScope as j, SyncStateFilter as k, ListLiveFilter as l, ParquetCodec as m, CompactionTier as n, FileSetRef as o, ManifestStore as p, enumeratePartitions as q, DEFAULT_SEARCH_TYPE as r, GcCtx as s, CodecCtx as t, LockScope as u, QueryCtx as v, RunSQLOptions as w, QueryExecutor as x, QueryExecuteOptions as y, dayPartition as z };

package/dist/_chunks/storage.mjs

@@ -11,6 +11,9 @@ function inferLegacyTier(entry) {
 function dayPartition(date) {
 	return `daily/${date}`;
 }
+function hourPartition(date) {
+	return `hourly/${date}`;
+}
 function monthPartition(month) {
 	return `monthly/${month}`;
 }
@@ -36,4 +39,4 @@ function objectKey(ctx, table, partition, version, searchType) {
 function tenantPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/` : `u_${ctx.userId}/`;
 }
-export { mondayOfWeek as a, quarterOfMonth as c, weekPartition as d, inferSearchType as i, quarterPartition as l, dayPartition as n, monthPartition as o, inferLegacyTier as r, objectKey as s, DEFAULT_SEARCH_TYPE as t, tenantPrefix as u };
+export { inferSearchType as a, objectKey as c, tenantPrefix as d, weekPartition as f, inferLegacyTier as i, quarterOfMonth as l, dayPartition as n, mondayOfWeek as o, hourPartition as r, monthPartition as s, DEFAULT_SEARCH_TYPE as t, quarterPartition as u };

package/dist/adapters/filesystem.d.mts

@@ -1,4 +1,4 @@
-import { f as ManifestStore, i as DataSource } from "../_chunks/storage.mjs";
+import { i as DataSource, p as ManifestStore } from "../_chunks/storage.mjs";
 interface FilesystemDataSourceOptions {
   rootDir: string;
 }

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { i as inferSearchType, r as inferLegacyTier } from "../_chunks/storage.mjs";
+import { a as inferSearchType, i as inferLegacyTier } from "../_chunks/storage.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";

package/dist/adapters/hyparquet.d.mts

@@ -1,4 +1,4 @@
-import { S as Row, i as DataSource, j as TableName, p as ParquetCodec, t as CodecCtx } from "../_chunks/storage.mjs";
+import { C as Row, M as TableName, i as DataSource, m as ParquetCodec, t as CodecCtx } from "../_chunks/storage.mjs";
 import { t as ColumnDef } from "../_chunks/schema.mjs";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
 interface EncodeFlexOptions {

package/dist/adapters/node.d.mts

@@ -1,4 +1,4 @@
-import { T as StorageEngine, i as DataSource } from "../_chunks/storage.mjs";
+import { E as StorageEngine, i as DataSource } from "../_chunks/storage.mjs";
 import { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { t as SnapshotIndex } from "../_chunks/snapshot.mjs";
 import { Row, TableName } from "@gscdump/contracts";

package/dist/adapters/node.mjs

@@ -1,4 +1,4 @@
-import { a as createDuckDBExecutor, i as createDuckDBCodec, n as createStorageEngine } from "../_chunks/engine.mjs";
+import { a as createDuckDBCodec, n as createStorageEngine, o as createDuckDBExecutor } from "../_chunks/engine.mjs";
 import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
 import path from "node:path";

package/dist/adapters/r2-manifest.d.mts

@@ -1,4 +1,4 @@
-import { f as ManifestStore, j as TableName } from "../_chunks/storage.mjs";
+import { M as TableName, p as ManifestStore } from "../_chunks/storage.mjs";
 interface R2ObjectMetadata {
   etag: string;
 }

package/dist/adapters/r2-manifest.mjs

@@ -1,4 +1,4 @@
-import { i as inferSearchType, r as inferLegacyTier } from "../_chunks/storage.mjs";
+import { a as inferSearchType, i as inferLegacyTier } from "../_chunks/storage.mjs";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 function defaultSnapshotId() {
 	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;

package/dist/contracts.d.mts

@@ -1,2 +1,2 @@
-import { A as SyncStateScope, C as RunSQLOptions, D as SyncStateDetail, E as SyncState, F as WatermarkScope, I as WriteCtx, L as WriteResult, M as TenantCtx, N as Watermark, O as SyncStateFilter, P as WatermarkFilter, S as Row, T as StorageEngine, _ as QueryCtx, a as EngineOptions, b as QueryExecutor, c as ListLiveFilter, f as ManifestStore, i as DataSource, j as TableName, k as SyncStateKind, l as LockScope, n as CompactionTier, o as FileSetRef, p as ParquetCodec, s as GcCtx, t as CodecCtx, u as ManifestEntry, v as QueryExecuteOptions, w as SearchType, x as QueryResult, y as QueryExecuteResult } from "./_chunks/storage.mjs";
-export { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };
+import { A as SyncStateKind, C as Row, D as SyncState, E as StorageEngine, F as WatermarkFilter, I as WatermarkScope, L as WriteCtx, M as TableName, N as TenantCtx, O as SyncStateDetail, P as Watermark, R as WriteResult, S as QueryResult, T as SearchType, a as EngineOptions, b as QueryExecuteResult, d as ManifestEntry, i as DataSource, j as SyncStateScope, k as SyncStateFilter, l as ListLiveFilter, m as ParquetCodec, n as CompactionTier, o as FileSetRef, p as ManifestStore, s as GcCtx, t as CodecCtx, u as LockScope, v as QueryCtx, w as RunSQLOptions, x as QueryExecutor, y as QueryExecuteOptions } from "./_chunks/storage.mjs";
+export type { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };

package/dist/index.d.mts

@@ -1,21 +1,49 @@
-import { A as SyncStateScope, B as inferSearchType, C as RunSQLOptions, D as SyncStateDetail, E as SyncState, F as WatermarkScope, G as enumeratePartitions, H as CompactionThresholds, I as WriteCtx, L as WriteResult, M as TenantCtx, N as Watermark, O as SyncStateFilter, P as WatermarkFilter, R as dayPartition, S as Row, T as StorageEngine, U as RAW_DAILY_COMPACT_THRESHOLD, V as objectKey, W as countRawDailies, _ as QueryCtx, a as EngineOptions, b as QueryExecutor, c as ListLiveFilter, d as ManifestPurgeResult, f as ManifestStore, g as PurgeUrlsResult, h as PurgeResult, i as DataSource, j as TableName, k as SyncStateKind, l as LockScope, m as PurgeFilter, n as CompactionTier, o as FileSetRef, p as ParquetCodec, r as DEFAULT_SEARCH_TYPE, s as GcCtx, t as CodecCtx, u as ManifestEntry, v as QueryExecuteOptions, w as SearchType, x as QueryResult, y as QueryExecuteResult, z as inferLegacyTier } from "./_chunks/storage.mjs";
+import { A as SyncStateKind, B as hourPartition, C as Row, D as SyncState, E as StorageEngine, F as WatermarkFilter, G as RAW_DAILY_COMPACT_THRESHOLD, H as inferSearchType, I as WatermarkScope, K as countRawDailies, L as WriteCtx, M as TableName, N as TenantCtx, O as SyncStateDetail, P as Watermark, R as WriteResult, S as QueryResult, T as SearchType, U as objectKey, V as inferLegacyTier, W as CompactionThresholds, _ as PurgeUrlsResult, a as EngineOptions, b as QueryExecuteResult, c as Grain, d as ManifestEntry, f as ManifestPurgeResult, g as PurgeResult, h as PurgeFilter, i as DataSource, j as SyncStateScope, k as SyncStateFilter, l as ListLiveFilter, m as ParquetCodec, n as CompactionTier, o as FileSetRef, p as ManifestStore, q as enumeratePartitions, r as DEFAULT_SEARCH_TYPE, s as GcCtx, t as CodecCtx, u as LockScope, v as QueryCtx, w as RunSQLOptions, x as QueryExecutor, y as QueryExecuteOptions, z as dayPartition } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
-import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
+import { _ as page_keywords, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as keywords, h as hourly_pages, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema, v as pages } from "./_chunks/schema.mjs";
 import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { Row as Row$1, TableName as TableName$1 } from "@gscdump/contracts";
+import { Grain as Grain$1, Row as Row$1, TableName as TableName$1 } from "@gscdump/contracts";
 declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
+interface GcDeps {
+  dataSource: DataSource;
+  manifestStore: ManifestStore;
+}
+interface GcOptions {
+  userId?: string;
+  siteId?: string;
+  /**
+   * Retention for hourly partitions (`hourly/{date}`) in milliseconds.
+   * Defaults to 90 days; entries with `createdAt < now - hourlyRetentionMs`
+   * are retired and their bytes deleted alongside ordinary orphan sweeping.
+   */
+  hourlyRetentionMs?: number;
+}
+declare function gcOrphansImpl(deps: GcDeps, now: number, graceMs: number, opts?: GcOptions): Promise<{
+  deleted: number;
+}>;
 interface IngestAccumulatorEngine {
   writeDay: (scope: TenantCtx & {
     table: TableName$1;
     date: string;
     searchType?: SearchType;
   }, rows: Row$1[]) => Promise<void>;
+  /**
+   * Routed when the accumulator's `ctx.grain === 'hour'`. Same scope shape as
+   * `writeDay`; `date` is the PT calendar day, rows carry `hour` + `date`.
+   * Optional so hosts that never opt into hourly need not implement it.
+   */
+  writeHour?: (scope: TenantCtx & {
+    table: TableName$1;
+    date: string;
+    searchType?: SearchType;
+  }, rows: Row$1[]) => Promise<void>;
   setSyncState: (scope: TenantCtx & {
     table: TableName$1;
     date: string;
@@ -28,6 +56,12 @@ interface IngestAccumulatorCtx {
   userId: string | number;
   siteId: string;
   searchType?: SearchType;
+  /**
+   * Temporal granularity for this accumulator. `'day'` (default) routes
+   * flushed buckets to `engine.writeDay`. `'hour'` routes to
+   * `engine.writeHour` and requires the engine implementation to be set.
+   */
+  grain?: Grain$1;
 }
 interface IngestAccumulatorHooks {
   /**
@@ -113,4 +147,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type GscApiRow, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, parseEnabledSearchTypes, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, parseEnabledSearchTypes, rebuildDailyFromHourly, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/index.mjs

@@ -1,11 +1,12 @@
 import { n as coerceRows, t as coerceRow } from "./_chunks/coerce.mjs";
-import { a as inferTable, c as countries, d as keywords, f as page_keywords, i as dimensionToColumn, l as devices, n as allTables, p as pages, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
-import { i as inferSearchType, n as dayPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
+import { a as inferTable, c as countries, d as hourly_pages, f as keywords, i as dimensionToColumn, l as devices, m as pages, n as allTables, p as page_keywords, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
+import { a as inferSearchType, c as objectKey, i as inferLegacyTier, n as dayPartition, r as hourPartition, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
 import { a as RAW_DAILY_COMPACT_THRESHOLD, c as enumeratePartitions, i as substituteNamedFiles, r as resolveToSQL, s as countRawDailies, t as FILES_PLACEHOLDER } from "./_chunks/compiler.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { a as createDuckDBExecutor, i as createDuckDBCodec, n as createStorageEngine, r as canonicalEmptyParquetSchema, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
+import { a as createDuckDBCodec, i as canonicalEmptyParquetSchema, n as createStorageEngine, o as createDuckDBExecutor, r as gcOrphansImpl, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
 import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
+import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 const NOOP_RESULT = {
 	flushed: 0,
@@ -37,7 +38,7 @@ function createIngestAccumulator(opts) {
 	const acc = createRowAccumulator(accOpts);
 	async function writeOne(table, date, rows) {
 		const scope = scopeOf(ctx, table, date);
-		return engine.writeDay(scope, rows).then(() => engine.setSyncState(scope, "done")).then(async () => {
+		return (ctx.grain === "hour" ? engine.writeHour ?? (() => Promise.reject(/* @__PURE__ */ new Error("ingest accumulator: grain=hour requires engine.writeHour"))) : engine.writeDay)(scope, rows).then(() => engine.setSyncState(scope, "done")).then(async () => {
 			await hooks.onWritten?.({
 				table,
 				date,
@@ -214,4 +215,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, parseEnabledSearchTypes, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, parseEnabledSearchTypes, rebuildDailyFromHourly, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/ingest.d.mts

@@ -1,4 +1,4 @@
-import { S as Row, j as TableName } from "./_chunks/storage.mjs";
+import { C as Row, M as TableName } from "./_chunks/storage.mjs";
 /**
  * Canonical GSC API dimension order per table. Consumers hitting the raw
  * `searchanalytics.query` endpoint must request dimensions in this order so

package/dist/ingest.mjs

@@ -8,7 +8,8 @@ const TABLE_DIMS = {
 		"query",
 		"date"
 	],
-	search_appearance: ["searchAppearance", "date"]
+	search_appearance: ["searchAppearance", "date"],
+	hourly_pages: ["hour", "page"]
 };
 function toPath(gscUrl) {
 	try {
@@ -80,6 +81,21 @@ function transformGscRow(table, apiRow, options = {}) {
 			}
 		};
 	}
+	if (table === "hourly_pages") {
+		const hour = String(keys[0] ?? "");
+		const date = hour.slice(0, 10);
+		return {
+			date,
+			row: {
+				url: toPath(String(keys[1] ?? "")),
+				hour,
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
 	if (table === "search_appearance") {
 		const date = String(keys[1] ?? "");
 		return {

package/dist/planner.d.mts

@@ -1,3 +1,3 @@
-import { G as enumeratePartitions } from "./_chunks/storage.mjs";
+import { q as enumeratePartitions } from "./_chunks/storage.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, r as compileLogicalQueryPlan, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
-export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };
+export { FILES_PLACEHOLDER, type ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/resolver/index.d.mts

@@ -1,8 +1,8 @@
-import { j as TableName$1, w as SearchType$1 } from "../_chunks/storage.mjs";
+import { M as TableName$1, T as SearchType$1 } from "../_chunks/storage.mjs";
 import { a as ResolvedSQLOptimized, i as ResolvedSQL, n as ExtraQuery, o as ResolverAdapter, r as ResolvedComparisonSQL, s as ResolverOptions, t as ComparisonFilter } from "../_chunks/types.mjs";
 import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities } from "gscdump/query/plan";
 import { SQL } from "drizzle-orm";
-import { TableName } from "@gscdump/contracts";
+import { Grain, TableName } from "@gscdump/contracts";
 import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
 type DimensionSurface = 'api' | 'stored' | 'derived';
 interface DimensionBinding {
@@ -102,6 +102,14 @@ interface RunQueryCtx {
   siteId: string;
   table: TableName$1;
   searchType?: SearchType$1;
+  /**
+   * Temporal granularity. `'day'` (default) drives `enumeratePartitions` to
+   * emit `daily/{date}` only; hourly partitions are skipped by construction.
+   * `'hour'` is reserved for hourly read paths and must use the dedicated
+   * hourly query surface (callers pass `partitions: [hourly/{date}]`
+   * directly through `runSQL`).
+   */
+  grain?: Grain;
 }
 interface RunSQLFn {
   (opts: {

package/dist/rollups.d.mts

@@ -1,8 +1,7 @@
-import { i as DataSource, o as FileSetRef } from "./_chunks/storage.mjs";
+import { C as Row$1, i as DataSource, o as FileSetRef } from "./_chunks/storage.mjs";
 import { t as ColumnDef } from "./_chunks/schema.mjs";
 import { TenantCtx } from "@gscdump/contracts";
 import { SearchType } from "gscdump/query";
-import * as _$_gscdump_engine_contracts0 from "@gscdump/engine/contracts";
 interface RollupCtx extends TenantCtx {
   /** When the rollup was built. Stamped into payload + filename. */
   builtAt: number;
@@ -15,7 +14,7 @@ interface RollupEngine {
   runSQL: (opts: {
     ctx: TenantCtx;
     fileSets: Record<string, FileSetRef>;
-    table?: _$_gscdump_engine_contracts0.TableName;
+    table?: import('@gscdump/engine/contracts').TableName;
     sql: string;
     params?: unknown[];
     /**
@@ -26,7 +25,7 @@ interface RollupEngine {
      */
     searchType?: SearchType;
   }) => Promise<{
-    rows: _$_gscdump_engine_contracts0.Row[];
+    rows: import('@gscdump/engine/contracts').Row[];
   }>;
 }
 /**
@@ -235,5 +234,37 @@ declare const sitemapHealthRollup: RollupDef;
  * exist on the site.
  */
 declare const sitemapChanges28dRollup: RollupDef;
+/**
+ * Aggregate one day's `hourly_pages` partition into the daily `pages` shape
+ * and write it to the daily Discover partition. After this runs for date D,
+ * the daily query path serves D from `pages/.../daily/D` and the `hourly/D`
+ * partition becomes read-only / GC-only.
+ *
+ * `(position - 1)` weighting matches the storage convention encoded by
+ * `toSumPosition`: `sum_position = SUM((position - 1) * impressions)`, so a
+ * downstream `SUM(sum_position) / SUM(impressions) + 1` recovers the mean.
+ *
+ * searchType-scoped: only call with `searchType: 'discover'`. The hourly
+ * partition lives under `hourly_pages` and the output lands under `pages` so
+ * existing dashboard queries (which read `pages`) see the rolled-up day
+ * transparently.
+ */
+interface RebuildDailyFromHourlyOptions {
+  engine: RollupEngine & {
+    writeDay: (scope: TenantCtx & {
+      table: TableTypeName;
+      date: string;
+      searchType?: SearchType;
+    }, rows: Row$1[]) => Promise<void>;
+  };
+  ctx: TenantCtx;
+  /** PT calendar day to roll up. */
+  date: string;
+  searchType: 'discover';
+}
+type TableTypeName = import('@gscdump/contracts').TableName;
+declare function rebuildDailyFromHourly(opts: RebuildDailyFromHourlyOptions): Promise<{
+  rowsWritten: number;
+}>;
 declare const DEFAULT_ROLLUPS: readonly RollupDef[];
-export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -577,6 +577,43 @@ const sitemapChanges28dRollup = {
 		};
 	}
 };
+async function rebuildDailyFromHourly(opts) {
+	const { engine, ctx, date, searchType } = opts;
+	const rows = (await engine.runSQL({
+		ctx,
+		table: "hourly_pages",
+		fileSets: { FILES: {
+			table: "hourly_pages",
+			partitions: [`hourly/${date}`]
+		} },
+		searchType,
+		sql: `
+      SELECT
+        url,
+        DATE '${date}' AS date,
+        SUM(clicks)::BIGINT AS clicks,
+        SUM(impressions)::BIGINT AS impressions,
+        SUM(sum_position)::DOUBLE AS sum_position
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date = '${date}'
+      GROUP BY url
+    `
+	})).rows.map((r) => ({
+		url: r.url,
+		date,
+		clicks: Number(r.clicks),
+		impressions: Number(r.impressions),
+		sum_position: Number(r.sum_position)
+	}));
+	await engine.writeDay({
+		userId: ctx.userId,
+		siteId: ctx.siteId,
+		table: "pages",
+		date,
+		searchType
+	}, rows);
+	return { rowsWritten: rows.length };
+}
 const DEFAULT_ROLLUPS = [
 	dailyTotalsRollup,
 	weeklyTotalsRollup,
@@ -589,4 +626,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapHealthRollup,
 	sitemapChanges28dRollup
 ];
-export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/schema.d.mts

@@ -1,2 +1,2 @@
-import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, l as schemaFor, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema, v as search_appearance } from "./_chunks/schema.mjs";
-export { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, devices, dimensionToColumn, drizzleSchema, inferTable, keywords, page_keywords, pages, schemaFor, search_appearance };
+import { _ as page_keywords, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as keywords, h as hourly_pages, i as TableSchema, l as schemaFor, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema, v as pages, y as search_appearance } from "./_chunks/schema.mjs";
+export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, devices, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, keywords, page_keywords, pages, schemaFor, search_appearance };

package/dist/schema.mjs

@@ -1,2 +1,2 @@
-import { a as inferTable, c as countries, d as keywords, f as page_keywords, i as dimensionToColumn, l as devices, m as search_appearance, n as allTables, o as schemaFor, p as pages, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, devices, dimensionToColumn, drizzleSchema, inferTable, keywords, page_keywords, pages, schemaFor, search_appearance };
+import { a as inferTable, c as countries, d as hourly_pages, f as keywords, h as search_appearance, i as dimensionToColumn, l as devices, m as pages, n as allTables, o as schemaFor, p as page_keywords, r as currentSchemaVersion, s as TABLE_METADATA, t as SCHEMAS, u as drizzleSchema } from "./_chunks/schema.mjs";
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, devices, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, keywords, page_keywords, pages, schemaFor, search_appearance };

package/dist/scope.d.mts

@@ -16,6 +16,12 @@ interface ScopedRunnerOptions {
   startDate?: string;
   /** Inclusive upper bound for `date`. Ignored if `window` is supplied. */
   endDate?: string;
+  /**
+   * Temporal granularity. `'day'` (default) filters on `table.date`. `'hour'`
+   *  filters on `table.hour` when the table exposes that column (e.g.
+   *  `hourly_pages`); falls back to date filtering otherwise.
+   */
+  grain?: 'day' | 'hour';
 }
 interface TableScope {
   wherePredicates: SQL[];

package/dist/scope.mjs

@@ -2,11 +2,13 @@ import { and, eq, gte, lte } from "drizzle-orm";
 function buildTableScope(table, opts) {
 	const predicates = [];
 	if (opts.siteId && "site_id" in table) predicates.push(eq(table.site_id, opts.siteId));
-	if ("date" in table) {
+	const useHour = (opts.grain ?? "day") === "hour" && "hour" in table;
+	const filterCol = useHour ? table.hour : table.date;
+	if ("date" in table || useHour) {
 		const start = opts.window?.start ?? opts.startDate;
 		const end = opts.window?.end ?? opts.endDate;
-		if (start) predicates.push(gte(table.date, start));
-		if (end) predicates.push(lte(table.date, end));
+		if (start) predicates.push(gte(filterCol, start));
+		if (end) predicates.push(lte(filterCol, end));
 	}
 	return {
 		wherePredicates: predicates,

package/dist/source/index.d.mts

@@ -1,4 +1,4 @@
-import { M as TenantCtx, S as Row, T as StorageEngine, w as SearchType$1 } from "../_chunks/storage.mjs";
+import { C as Row, E as StorageEngine, N as TenantCtx, T as SearchType$1 } from "../_chunks/storage.mjs";
 import { n as AnalysisResult, t as AnalysisParams } from "../_chunks/analysis-types.mjs";
 import { o as ResolverAdapter } from "../_chunks/types.mjs";
 import { C as ExecuteSqlOptions, E as SourceCapabilities, S as AnalysisSourceKind, T as QueryRow, t as AnalyzerRegistry, w as FileSet, x as AnalysisQuerySource } from "../_chunks/registry.mjs";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.17.5",
+  "version": "0.18.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.17.5",
-    "gscdump": "0.17.5"
+    "gscdump": "0.18.1",
+    "@gscdump/contracts": "0.18.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",