@gscdump/engine 0.15.0 → 0.17.0

package/dist/_chunks/engine.mjs

@@ -274,12 +274,13 @@ function createStorageEngine(opts) {
 			table: ctx.table,
 			partition
 		}, async () => {
-			const superseding = (await manifestStore.listLive({
+			const superseding = await manifestStore.listLive({
 				userId: ctx.userId,
 				siteId: ctx.siteId,
 				table: ctx.table,
-				partitions: [partition]
-			})).filter((e) => inferSearchType(e) === inferSearchType({ searchType }));
+				partitions: [partition],
+				searchType: inferSearchType({ searchType })
+			});
 			const normalizedRows = rows.map((r) => normalizeRow(ctx.table, r));
 			const key = objectKey(ctx, ctx.table, partition, now, searchType);
 			const { bytes: writtenBytes, rowCount } = await codec.writeRows({ table: ctx.table }, normalizedRows, key, dataSource);
@@ -322,7 +323,8 @@ function createStorageEngine(opts) {
 				userId: opts.ctx.userId,
 				siteId: opts.ctx.siteId,
 				table: ref.table,
-				partitions: ref.partitions
+				partitions: ref.partitions,
+				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
 			})).map((e) => e.objectKey)];
 		}));
 		opts.signal?.throwIfAborted();
@@ -368,7 +370,8 @@ function createStorageEngine(opts) {
 			} },
 			sql: resolved.sql,
 			params: resolved.params,
-			signal: ctx.signal
+			signal: ctx.signal,
+			...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
 		});
 	}
 	async function compactTiered(ctx, thresholds) {

package/dist/_chunks/storage.d.mts

@@ -33,6 +33,13 @@ interface WriteCtx extends TenantCtx {
 interface QueryCtx extends TenantCtx {
   table?: TableName;
   signal?: AbortSignal;
+  /**
+   * Restrict the query to a single GSC search-type partition (`web`,
+   * `discover`, etc.). Undefined preserves the cross-type union for
+   * legacy/web-only deployments; explicit value scopes the read to
+   * manifest entries written for that type. Mirrors {@link WriteCtx.searchType}.
+   */
+  searchType?: SearchType;
 }
 interface GcCtx {
   now?: () => number;
@@ -103,6 +110,18 @@ interface ListLiveFilter {
    * an explicit `tier` field match on {@link inferLegacyTier}.
    */
   tier?: CompactionTier;
+  /**
+   * Narrow to a single GSC searchType slice. Undefined means "no filter" — used
+   * by cross-type admin paths (GC / orphan sweep, tenant-stats site discovery).
+   * Explicit value filters to that slice; pass `'web'` to match the legacy /
+   * sentinel-`''` entries via {@link inferSearchType}.
+   *
+   * Read paths that scope to a single (user, site, table) cohort MUST set this
+   * once writes from multiple search types coexist for that cohort, otherwise
+   * the result unions web + non-web entries into a single query and double-
+   * counts metrics.
+   */
+  searchType?: SearchType;
 }
 interface DataSource {
   read: (key: string, range?: {
@@ -379,6 +398,14 @@ interface RunSQLOptions {
   sql: string;
   params?: unknown[];
   signal?: AbortSignal;
+  /**
+   * Restrict every manifest lookup the runner performs to a single
+   * search-type slice. Applies uniformly across all `fileSets`; per-
+   * fileSet overrides aren't supported (the only multi-fileSet caller,
+   * comparison joins, always wants the same slice for both windows).
+   * Undefined keeps the legacy cross-type union.
+   */
+  searchType?: SearchType;
 }
 interface StorageEngine {
   writeDay: (ctx: WriteCtx, rows: Row[]) => Promise<void>;

package/dist/adapters/filesystem.mjs

@@ -119,6 +119,7 @@ function matchesFilter(entry, filter) {
 	if (filter.table !== void 0 && entry.table !== filter.table) return false;
 	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
 	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
 	return true;
 }
 function lockFileFor(locksDir, scope) {

package/dist/adapters/node.d.mts

@@ -2,6 +2,7 @@ import { T as StorageEngine, i as DataSource } from "../_chunks/storage.mjs";
 import { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { t as SnapshotIndex } from "../_chunks/snapshot.mjs";
 import { Row, TableName } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */
@@ -24,6 +25,11 @@ interface NodeHarness {
     siteUrl: string;
     table: TableName;
     params?: unknown[];
+    /**
+     * Restrict the underlying manifest lookup to a single GSC search-type
+     * slice. Undefined keeps the legacy cross-type union.
+     */
+    searchType?: SearchType;
   }) => Promise<{
     rows: Row[];
     sql: string;

package/dist/adapters/node.mjs

@@ -25,7 +25,8 @@ function createNodeHarness(opts) {
 			table: runOpts.table,
 			fileSets: { FILES: { table: runOpts.table } },
 			sql: runOpts.sql,
-			params: runOpts.params ?? []
+			params: runOpts.params ?? [],
+			...runOpts.searchType !== void 0 ? { searchType: runOpts.searchType } : {}
 		});
 		return {
 			rows: result.rows,

package/dist/adapters/r2-manifest.mjs

@@ -33,6 +33,7 @@ function matchesEntryFilter(entry, filter) {
 	if (filter.table !== void 0 && entry.table !== filter.table) return false;
 	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
 	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
+	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
 	return true;
 }
 function matchesWatermarkFilter(w, filter) {

package/dist/rollups.d.mts

@@ -1,6 +1,7 @@
 import { i as DataSource, o as FileSetRef } from "./_chunks/storage.mjs";
 import { t as ColumnDef } from "./_chunks/schema.mjs";
 import { TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 import * as _$_gscdump_engine_contracts0 from "@gscdump/engine/contracts";
 interface RollupCtx extends TenantCtx {
   /** When the rollup was built. Stamped into payload + filename. */
@@ -17,6 +18,13 @@ interface RollupEngine {
     table?: _$_gscdump_engine_contracts0.TableName;
     sql: string;
     params?: unknown[];
+    /**
+     * Restrict every manifest lookup to a single GSC search-type slice. The
+     * rollup runner forwards `RebuildRollupsOptions.searchType` so the
+     * aggregated facts never mix web + non-web rows. Undefined preserves
+     * the legacy cross-type union (web-only tenants).
+     */
+    searchType?: SearchType;
   }) => Promise<{
     rows: _$_gscdump_engine_contracts0.Row[];
   }>;
@@ -67,6 +75,13 @@ interface RollupDef {
      * lives in ICU).
      */
     builtAt: number;
+    /**
+     * GSC search-type slice the runner was invoked for. Builders forward
+     * this to every `engine.runSQL` call so the aggregated facts come
+     * from one cohort. Undefined preserves the legacy cross-type union
+     * (used by web-only tenants and admin paths).
+     */
+    searchType?: SearchType;
   }) => Promise<unknown>;
 }
 /**
@@ -85,14 +100,24 @@ interface ParquetRollupPointer {
   parquetKey: string;
   rowCount: number;
 }
-declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number): string;
-declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number): string;
+declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;
+declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;
 interface RebuildRollupsOptions {
   engine: RollupEngine;
   dataSource: DataSource;
   ctx: TenantCtx;
   defs: readonly RollupDef[];
   now?: () => number;
+  /**
+   * Build rollups for a single GSC search-type slice. Threads into every
+   * builder's `engine.runSQL` call so the aggregated facts come from one
+   * cohort, and namespaces the output object keys under a `<searchType>/`
+   * segment so per-slice rollups coexist without overwriting each other.
+   * Undefined preserves the legacy cross-type behaviour (one rollup over
+   * the union of all slices, written to the legacy path) — fine for web-
+   * only tenants and explicit cross-type admin views.
+   */
+  searchType?: SearchType;
 }
 interface RebuildRollupResult {
   id: string;

package/dist/rollups.mjs

@@ -1,25 +1,28 @@
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexKey } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
-function rollupPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
+function rollupPrefix(ctx, searchType) {
+	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
+	return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
 }
-function rollupKey(ctx, id, builtAt) {
-	return `${rollupPrefix(ctx)}/${id}__v${builtAt}.json`;
+function rollupKey(ctx, id, builtAt, searchType) {
+	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.json`;
 }
-function rollupParquetKey(ctx, id, builtAt) {
-	return `${rollupPrefix(ctx)}/${id}__v${builtAt}.parquet`;
+function rollupParquetKey(ctx, id, builtAt, searchType) {
+	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.parquet`;
 }
 async function rebuildRollups(opts) {
 	const now = opts.now ?? (() => Date.now());
 	const results = [];
+	const searchType = opts.searchType;
 	for (const def of opts.defs) {
 		const builtAt = now();
 		const payload = await def.build({
 			engine: opts.engine,
 			ctx: opts.ctx,
 			dataSource: opts.dataSource,
-			builtAt
+			builtAt,
+			...searchType !== void 0 ? { searchType } : {}
 		});
 		if (def.format === "parquet") {
 			if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
@@ -28,7 +31,7 @@ async function rebuildRollups(opts) {
 				columns: def.parquetColumns,
 				sortKey: def.parquetSortKey
 			});
-			const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt);
+			const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt, searchType);
 			await opts.dataSource.write(parquetKey, parquetBytes);
 			const pointer = {
 				parquetKey,
@@ -42,7 +45,7 @@ async function rebuildRollups(opts) {
 				payload: pointer
 			};
 			const envelopeBytes = new TextEncoder().encode(JSON.stringify(envelope));
-			const key = rollupKey(opts.ctx, def.id, builtAt);
+			const key = rollupKey(opts.ctx, def.id, builtAt, searchType);
 			await opts.dataSource.write(key, envelopeBytes);
 			results.push({
 				id: def.id,
@@ -63,7 +66,7 @@ async function rebuildRollups(opts) {
 		};
 		const json = JSON.stringify(envelope);
 		const bytes = new TextEncoder().encode(json);
-		const key = rollupKey(opts.ctx, def.id, builtAt);
+		const key = rollupKey(opts.ctx, def.id, builtAt, searchType);
 		await opts.dataSource.write(key, bytes);
 		results.push({
 			id: def.id,
@@ -81,7 +84,7 @@ function utcDateMinusDays(at, days) {
 const dailyTotalsRollup = {
 	id: "daily_totals",
 	windowDays: null,
-	async build({ engine, ctx }) {
+	async build({ engine, ctx, searchType }) {
 		const pages = await engine.runSQL({
 			ctx,
 			table: "pages",
@@ -95,7 +98,8 @@ const dailyTotalsRollup = {
         FROM read_parquet({{FILES}}, union_by_name = true)
         GROUP BY date
         ORDER BY date
-      `
+      `,
+			...searchType !== void 0 ? { searchType } : {}
 		});
 		const keywords = await engine.runSQL({
 			ctx,
@@ -107,7 +111,8 @@ const dailyTotalsRollup = {
           SUM(impressions)::BIGINT AS impressions
         FROM read_parquet({{FILES}}, union_by_name = true)
         GROUP BY date
-      `
+      `,
+			...searchType !== void 0 ? { searchType } : {}
 		});
 		const keywordImpressionsByDate = /* @__PURE__ */ new Map();
 		for (const r of keywords.rows) keywordImpressionsByDate.set(String(r.date), BigInt(r.impressions));
@@ -128,11 +133,12 @@ const dailyTotalsRollup = {
 const weeklyTotalsRollup = {
 	id: "weekly_totals",
 	windowDays: null,
-	async build({ engine, ctx }) {
+	async build({ engine, ctx, searchType }) {
 		return (await engine.runSQL({
 			ctx,
 			table: "pages",
 			fileSets: { FILES: { table: "pages" } },
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           strftime(date_trunc('week', date::DATE), '%Y-%m-%d') AS week,
@@ -154,12 +160,13 @@ const weeklyTotalsRollup = {
 const topPages28dRollup = {
 	id: "top_pages_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt }) {
+	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
 		return (await engine.runSQL({
 			ctx,
 			table: "pages",
 			fileSets: { FILES: { table: "pages" } },
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           url,
@@ -183,12 +190,13 @@ const topPages28dRollup = {
 const topCountries28dRollup = {
 	id: "top_countries_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt }) {
+	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
 		return (await engine.runSQL({
 			ctx,
 			table: "countries",
 			fileSets: { FILES: { table: "countries" } },
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           country,
@@ -212,12 +220,13 @@ const topCountries28dRollup = {
 const topKeywords28dRollup = {
 	id: "top_keywords_28d",
 	windowDays: 28,
-	async build({ engine, ctx, builtAt }) {
+	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
 		return (await engine.runSQL({
 			ctx,
 			table: "keywords",
 			fileSets: { FILES: { table: "keywords" } },
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           query,
@@ -265,12 +274,13 @@ const topKeywords28dParquetRollup = {
 		}
 	],
 	parquetSortKey: ["clicks"],
-	async build({ engine, ctx, builtAt }) {
+	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
 		return (await engine.runSQL({
 			ctx,
 			table: "keywords",
 			fileSets: { FILES: { table: "keywords" } },
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           query,
@@ -381,7 +391,7 @@ const indexingHealthRollup = {
 const indexPercentRollup = {
 	id: "index_percent",
 	windowDays: 90,
-	async build({ engine, ctx, dataSource, builtAt }) {
+	async build({ engine, ctx, dataSource, builtAt, searchType }) {
 		const urlsKey = sitemapUrlsIndexKey(ctx);
 		if (!await dataSource.head?.(urlsKey)) return {
 			totalSitemapUrls: 0,
@@ -398,6 +408,7 @@ const indexPercentRollup = {
 					keys: [urlsKey]
 				}
 			},
+			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
           p.date AS date,

package/dist/source/index.d.mts

@@ -1,4 +1,4 @@
-import { M as TenantCtx, S as Row, T as StorageEngine } from "../_chunks/storage.mjs";
+import { M as TenantCtx, S as Row, T as StorageEngine, w as SearchType$1 } from "../_chunks/storage.mjs";
 import { n as AnalysisResult, t as AnalysisParams } from "../_chunks/analysis-types.mjs";
 import { o as ResolverAdapter } from "../_chunks/types.mjs";
 import { C as ExecuteSqlOptions, E as SourceCapabilities, S as AnalysisSourceKind, T as QueryRow, t as AnalyzerRegistry, w as FileSet, x as AnalysisQuerySource } from "../_chunks/registry.mjs";
@@ -72,6 +72,14 @@ declare const ENGINE_QUERY_CAPABILITIES: PlannerCapabilities;
 interface EngineQuerySourceOptions {
   engine: StorageEngine;
   ctx: TenantCtx;
+  /**
+   * Restrict every manifest lookup the source performs to a single search-type
+   * slice. Threads into `engine.query` and `engine.runSQL` so the wrapped
+   * source returns rows from one cohort instead of unioning web + non-web
+   * parquet. Undefined preserves legacy cross-type behaviour for web-only
+   * tenants and admin paths.
+   */
+  searchType?: SearchType$1;
 }
 /**
  * Wraps a storage engine as an `AnalysisQuerySource` with SQL execution.

package/dist/source/index.mjs

@@ -94,7 +94,7 @@ const ENGINE_SOURCE_CAPABILITIES = {
 	adapter: true
 };
 function createEngineQuerySource(options) {
-	const { engine, ctx } = options;
+	const { engine, ctx, searchType } = options;
 	return {
 		name: "engine",
 		kind: "local",
@@ -104,7 +104,10 @@ function createEngineQuerySource(options) {
 			const filterDims = getFilterDimensions(state.filter, isMetricDimension);
 			assertDimensionsSupported([...state.dimensions, ...filterDims], "stored", "engine query source");
 			if (state.dimensions.includes("queryCanonical") || filterDims.includes("queryCanonical")) throw new Error("engine query source does not support queryCanonical; use browser/sqlite query sources for derived dimensions");
-			return coerceRows((await engine.query(ctx, state)).rows);
+			return coerceRows((await engine.query({
+				...ctx,
+				...searchType !== void 0 ? { searchType } : {}
+			}, state)).rows);
 		},
 		async executeSql(sql, params, opts) {
 			const fileSets = opts?.fileSets;
@@ -114,7 +117,8 @@ function createEngineQuerySource(options) {
 				table: fileSets.FILES.table,
 				fileSets,
 				sql,
-				params: params ?? []
+				params: params ?? [],
+				...searchType !== void 0 ? { searchType } : {}
 			});
 			return coerceRows(rows);
 		}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.15.0",
+  "version": "0.17.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.15.0",
-    "gscdump": "0.15.0"
+    "@gscdump/contracts": "0.17.0",
+    "gscdump": "0.17.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -178,8 +178,8 @@
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.25.8",
     "hyparquet-writer": "^0.15.1",
-    "tsx": "^4.21.0",
-    "vitest": "^4.1.5"
+    "tsx": "^4.22.1",
+    "vitest": "^4.1.6"
   },
   "scripts": {
     "dev": "obuild --stub",