@gscdump/engine 0.7.5 → 0.8.0

package/dist/entities.d.mts

@@ -77,20 +77,6 @@ interface CreateInspectionStoreOptions {
   now?: () => number;
 }
 declare function createInspectionStore(opts: CreateInspectionStoreOptions): InspectionStore;
-interface InspectionSqlDriver {
-  exec: (sql: string) => void | Promise<void>;
-  run: (sql: string, params: unknown[]) => void | Promise<void>;
-  all: (sql: string, params: unknown[]) => unknown[] | Promise<unknown[]>;
-  serialize: () => Uint8Array | Promise<Uint8Array>;
-  close: () => void | Promise<void>;
-}
-interface CreateInspectionStoreSqliteOptions {
-  dataSource: DataSource;
-  openDriver: (bytes: Uint8Array | undefined) => InspectionSqlDriver | Promise<InspectionSqlDriver>;
-  hash?: (url: string) => string;
-}
-declare function inspectionSqliteKey(ctx: TenantCtx): string;
-declare function createInspectionStoreSqlite(opts: CreateInspectionStoreSqliteOptions): InspectionStore;
 /** GSC sitemap record we persist. Matches `Schema$WmxSitemap` but as plain JSON. */
 interface SitemapRecord {
   /** The sitemap URL (feedpath) as returned by GSC. */
@@ -189,4 +175,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateInspectionStoreSqliteOptions, CreateSitemapStoreOptions, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionSqlDriver, InspectionStore, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createInspectionStoreSqlite, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionSqliteKey, sitemapHistoryKey, sitemapIndexKey };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionStore, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };

package/dist/entities.mjs

@@ -78,156 +78,6 @@ function createInspectionStore(opts) {
 		}
 	};
 }
-function inspectionSqliteKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/inspections.db` : `u_${ctx.userId}/entities/inspections/inspections.db`;
-}
-const INSPECTION_SCHEMA_SQL = `
-CREATE TABLE IF NOT EXISTS inspections (
-  url_hash TEXT PRIMARY KEY,
-  url TEXT NOT NULL,
-  inspected_at TEXT NOT NULL,
-  index_status TEXT,
-  last_crawl_time TEXT,
-  google_canonical TEXT,
-  user_canonical TEXT,
-  coverage_state TEXT,
-  robots_txt_state TEXT,
-  indexing_state TEXT,
-  page_fetch_state TEXT,
-  mobile_usability_verdict TEXT,
-  rich_results_verdict TEXT,
-  raw TEXT
-);
-CREATE TABLE IF NOT EXISTS inspection_history (
-  year_month TEXT NOT NULL,
-  url_hash TEXT NOT NULL,
-  url TEXT NOT NULL,
-  inspected_at TEXT NOT NULL,
-  payload TEXT NOT NULL,
-  PRIMARY KEY (year_month, url_hash, inspected_at)
-);
-CREATE INDEX IF NOT EXISTS inspection_history_by_month ON inspection_history(year_month);
-`;
-function rowToRecord(r) {
-	const out = {
-		url: r.url,
-		inspectedAt: r.inspected_at
-	};
-	if (r.index_status != null) out.indexStatus = r.index_status;
-	if (r.last_crawl_time != null) out.lastCrawlTime = r.last_crawl_time;
-	if (r.google_canonical != null) out.googleCanonical = r.google_canonical;
-	if (r.user_canonical != null) out.userCanonical = r.user_canonical;
-	if (r.coverage_state != null) out.coverageState = r.coverage_state;
-	if (r.robots_txt_state != null) out.robotsTxtState = r.robots_txt_state;
-	if (r.indexing_state != null) out.indexingState = r.indexing_state;
-	if (r.page_fetch_state != null) out.pageFetchState = r.page_fetch_state;
-	if (r.mobile_usability_verdict != null) out.mobileUsabilityVerdict = r.mobile_usability_verdict;
-	if (r.rich_results_verdict != null) out.richResultsVerdict = r.rich_results_verdict;
-	if (r.raw != null) out.raw = JSON.parse(r.raw);
-	return out;
-}
-function shardForRecord(record) {
-	const m = YEAR_MONTH_RE.exec(record.inspectedAt);
-	return m ? `${m[1]}-${m[2]}` : "unknown";
-}
-function createInspectionStoreSqlite(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function withDriver(ctx, fn, persist) {
-		const key = inspectionSqliteKey(ctx);
-		const bytes = await ds.read(key).catch(() => void 0);
-		const driver = await opts.openDriver(bytes);
-		await driver.exec(INSPECTION_SCHEMA_SQL);
-		const result = await fn(driver);
-		if (persist) {
-			const out = await driver.serialize();
-			await ds.write(key, out);
-		}
-		await driver.close();
-		return result;
-	}
-	return {
-		async writeBatch(ctx, records) {
-			if (records.length === 0) return;
-			await withDriver(ctx, async (driver) => {
-				for (const r of records) {
-					const h = hash(r.url);
-					await driver.run(`INSERT INTO inspections (
-              url_hash, url, inspected_at, index_status, last_crawl_time,
-              google_canonical, user_canonical, coverage_state, robots_txt_state,
-              indexing_state, page_fetch_state, mobile_usability_verdict,
-              rich_results_verdict, raw
-            ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
-            ON CONFLICT(url_hash) DO UPDATE SET
-              url = excluded.url,
-              inspected_at = excluded.inspected_at,
-              index_status = excluded.index_status,
-              last_crawl_time = excluded.last_crawl_time,
-              google_canonical = excluded.google_canonical,
-              user_canonical = excluded.user_canonical,
-              coverage_state = excluded.coverage_state,
-              robots_txt_state = excluded.robots_txt_state,
-              indexing_state = excluded.indexing_state,
-              page_fetch_state = excluded.page_fetch_state,
-              mobile_usability_verdict = excluded.mobile_usability_verdict,
-              rich_results_verdict = excluded.rich_results_verdict,
-              raw = excluded.raw`, [
-						h,
-						r.url,
-						r.inspectedAt,
-						r.indexStatus ?? null,
-						r.lastCrawlTime ?? null,
-						r.googleCanonical ?? null,
-						r.userCanonical ?? null,
-						r.coverageState ?? null,
-						r.robotsTxtState ?? null,
-						r.indexingState ?? null,
-						r.pageFetchState ?? null,
-						r.mobileUsabilityVerdict ?? null,
-						r.richResultsVerdict ?? null,
-						r.raw === void 0 ? null : JSON.stringify(r.raw)
-					]);
-					await driver.run(`INSERT OR REPLACE INTO inspection_history
-               (year_month, url_hash, url, inspected_at, payload)
-               VALUES (?,?,?,?,?)`, [
-						shardForRecord(r),
-						h,
-						r.url,
-						r.inspectedAt,
-						JSON.stringify(r)
-					]);
-				}
-			}, true);
-		},
-		async getLatest(ctx, url) {
-			return await withDriver(ctx, async (driver) => {
-				const rows = await driver.all("SELECT * FROM inspections WHERE url_hash = ? LIMIT 1", [hash(url)]);
-				return rows.length === 0 ? void 0 : rowToRecord(rows[0]);
-			}, false);
-		},
-		async loadIndex(ctx) {
-			return await withDriver(ctx, async (driver) => {
-				const rows = await driver.all("SELECT * FROM inspections", []);
-				const records = {};
-				for (const r of rows) records[r.url_hash] = rowToRecord(r);
-				return {
-					version: 1,
-					records
-				};
-			}, false);
-		},
-		async loadHistory(ctx, yearMonth) {
-			return await withDriver(ctx, async (driver) => {
-				const rows = await driver.all("SELECT * FROM inspection_history WHERE year_month = ? ORDER BY inspected_at ASC", [yearMonth]);
-				if (rows.length === 0) return void 0;
-				return {
-					version: 1,
-					records: rows.map((r) => JSON.parse(r.payload))
-				};
-			}, false);
-		}
-	};
-}
 function sitemapIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
 }
@@ -356,4 +206,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createInspectionStoreSqlite, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionSqliteKey, sitemapHistoryKey, sitemapIndexKey };
+export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };

package/dist/index.d.mts

@@ -1,7 +1,6 @@
 import { A as SyncStateFilter, B as dayPartition, C as QueryResult, D as StorageEngine, E as SearchType, F as Watermark, G as objectKey, H as inferSearchType, I as WatermarkFilter, J as weekPartition, K as quarterOfMonth, L as WatermarkScope, M as SyncStateScope, N as TableName, O as SyncState, P as TenantCtx, R as WriteCtx, S as QueryExecutor, T as RunSQLOptions, U as mondayOfWeek, V as inferLegacyTier, W as monthPartition, X as enumeratePartitions, Y as CompactionThresholds, _ as PurgeResult, a as DataSource, b as QueryExecuteOptions, c as FileSetRef, d as LockScope, f as ManifestEntry, g as PurgeFilter, h as ParquetCodec, i as DEFAULT_SEARCH_TYPE, j as SyncStateKind, k as SyncStateDetail, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, q as quarterPartition, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeUrlsResult, w as Row, x as QueryExecuteResult, y as QueryCtx, z as WriteResult } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
 import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
-import { CreateInspectionStoreSqliteOptions, InspectionSqlDriver, createInspectionStoreSqlite, inspectionSqliteKey } from "./entities.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
@@ -10,4 +9,4 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, type CreateInspectionStoreSqliteOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type InspectionSqlDriver, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createInspectionStoreSqlite, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, inspectionSqliteKey, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };