@gscdump/engine 0.11.0 → 0.11.2

package/dist/_chunks/engine.mjs

@@ -318,6 +318,7 @@ function createStorageEngine(opts) {
 		opts.signal?.throwIfAborted();
 		const entries = Object.entries(opts.fileSets);
 		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
+			if (ref.keys !== void 0) return [name, ref.keys];
 			return [name, (await manifestStore.listLive({
 				userId: opts.ctx.userId,
 				siteId: opts.ctx.siteId,

package/dist/_chunks/storage.d.mts

@@ -375,6 +375,15 @@ interface QueryExecutor {
 interface FileSetRef {
   table: TableName;
   partitions?: string[];
+  /**
+   * Pre-resolved object keys, bypassing the manifest lookup. When provided,
+   * runSQL skips `manifestStore.listLive` for this entry and uses these keys
+   * directly. Use for entity-store sidecars (`entities/inspections/index.parquet`,
+   * `entities/sitemaps/urls/index.parquet`) which aren't registered in the
+   * analytics manifest. `table` is still required as the schema sentinel for
+   * the empty-fallback rewrite, but isn't consulted when `keys` is non-empty.
+   */
+  keys?: string[];
 }
 interface RunSQLOptions {
   ctx: TenantCtx;

package/dist/rollups.mjs

@@ -1,5 +1,5 @@
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "./entities.mjs";
+import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexKey } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -335,36 +335,35 @@ const indexingMetadataRollup = {
 		};
 	}
 };
-function sqlString(s) {
-	return `'${s.replace(/'/g, "''")}'`;
-}
 const indexingHealthRollup = {
 	id: "indexing_health",
 	windowDays: 90,
 	async build({ engine, ctx, dataSource, builtAt }) {
-		const uri = createInspectionStore({ dataSource }).parquetUri(ctx);
-		if (!uri) return { days: [] };
-		const cutoff = utcDateMinusDays(builtAt, 90);
+		const key = inspectionParquetKey(ctx);
+		if (!await dataSource.head?.(key)) return { days: [] };
 		const sql = `
       SELECT
-        substr(inspectedAt, 1, 10) AS date,
+        substr(CAST(inspectedAt AS VARCHAR), 1, 10) AS date,
         COUNT(*)::BIGINT AS total_urls,
-        SUM(CASE WHEN indexStatus = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS indexed_count,
-        SUM(CASE WHEN pageFetchState = 'SOFT_404' THEN 1 ELSE 0 END)::BIGINT AS soft_404,
-        SUM(CASE WHEN pageFetchState = 'REDIRECT_ERROR' THEN 1 ELSE 0 END)::BIGINT AS redirect,
-        SUM(CASE WHEN pageFetchState = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
-        SUM(CASE WHEN mobileUsabilityVerdict = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
-        SUM(CASE WHEN richResultsVerdict = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
-        SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND userCanonical <> googleCanonical THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
-      FROM read_parquet(${sqlString(uri)})
-      WHERE substr(inspectedAt, 1, 10) >= '${cutoff}'
+        SUM(CASE WHEN CAST(indexStatus AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS indexed_count,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'SOFT_404' THEN 1 ELSE 0 END)::BIGINT AS soft_404,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'REDIRECT_ERROR' THEN 1 ELSE 0 END)::BIGINT AS redirect,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
+        SUM(CASE WHEN CAST(mobileUsabilityVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
+        SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
+        SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND CAST(userCanonical AS VARCHAR) <> CAST(googleCanonical AS VARCHAR) THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
+      FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
+      WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(builtAt, 90)}'
       GROUP BY 1
       ORDER BY 1
     `;
 		return { days: (await engine.runSQL({
 			ctx,
 			table: "pages",
-			fileSets: {},
+			fileSets: { INSPECTIONS: {
+				table: "pages",
+				keys: [key]
+			} },
 			sql
 		})).rows.map((r) => ({
 			date: String(r.date),
@@ -383,8 +382,8 @@ const indexPercentRollup = {
 	id: "index_percent",
 	windowDays: 90,
 	async build({ engine, ctx, dataSource, builtAt }) {
-		const urlsUri = createSitemapStore({ dataSource }).urlsParquetUri(ctx);
-		if (!urlsUri) return {
+		const urlsKey = sitemapUrlsIndexKey(ctx);
+		if (!await dataSource.head?.(urlsKey)) return {
 			totalSitemapUrls: 0,
 			days: []
 		};
@@ -392,13 +391,19 @@ const indexPercentRollup = {
 		const numerator = await engine.runSQL({
 			ctx,
 			table: "pages",
-			fileSets: { PAGES: { table: "pages" } },
+			fileSets: {
+				PAGES: { table: "pages" },
+				URLS: {
+					table: "pages",
+					keys: [urlsKey]
+				}
+			},
 			sql: `
         SELECT
           p.date AS date,
           COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
         FROM read_parquet({{PAGES}}, union_by_name = true) p
-        INNER JOIN read_parquet(${sqlString(urlsUri)}) s
+        INNER JOIN read_parquet({{URLS}}, union_by_name = true) s
           ON s.loc = p.url AND s.removed_at IS NULL
         WHERE p.clicks > 0 AND p.date >= '${cutoff}'
         GROUP BY p.date
@@ -408,10 +413,13 @@ const indexPercentRollup = {
 		const denom = await engine.runSQL({
 			ctx,
 			table: "pages",
-			fileSets: {},
+			fileSets: { URLS: {
+				table: "pages",
+				keys: [urlsKey]
+			} },
 			sql: `
         SELECT COUNT(*)::BIGINT AS total
-        FROM read_parquet(${sqlString(urlsUri)})
+        FROM read_parquet({{URLS}}, union_by_name = true)
         WHERE removed_at IS NULL
       `
 		});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,7 +169,7 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.11.0"
+    "gscdump": "0.11.2"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",