@gscdump/analysis 0.33.0 → 0.33.4

package/dist/semantic/index.mjs

@@ -1,54 +1,8 @@
-var ContentGapSourceUnsupportedError = class extends Error {
-	constructor(kind) {
-		super(`content-gap requires a source with executeSql (got '${kind}'); use createBrowserQuerySource or createSqliteQuerySource`);
-		this.name = "ContentGapSourceUnsupportedError";
-	}
-};
-const ORIGIN_RE = /^https?:\/\/[^/]+/;
-const HTML_EXT_RE = /\.(html?|php|aspx?)$/i;
-const SEP_RE = /[-_]+/g;
-const DIGITS_ONLY_RE = /^\d+$/;
-const WWW_RE = /^www\./;
-const DOT_RE = /\./g;
-const HASH_RE = /#.*$/;
-const QUERY_RE = /\?.*$/;
-const TRAIL_SLASH_RE = /(?<=.)\/$/;
-const MODEL_ID = "Xenova/bge-base-en-v1.5";
-const QUERY_PREFIX = "Represent this sentence for searching relevant passages: ";
+const CONTENT_GAP_MODEL_ID = "Xenova/bge-base-en-v1.5";
+const CONTENT_GAP_QUERY_PREFIX = "Represent this sentence for searching relevant passages: ";
 const DB_NAME = "content-gap-embeddings";
 const STORE = "vectors";
 let dbPromise = null;
-function normalizeUrl(u) {
-	try {
-		const url = new URL(u);
-		url.hash = "";
-		url.search = "";
-		let path = url.pathname;
-		if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
-		return `${url.origin}${path}`;
-	} catch {
-		return u.replace(HASH_RE, "").replace(QUERY_RE, "").replace(TRAIL_SLASH_RE, "");
-	}
-}
-function deriveUrlText(url) {
-	try {
-		const u = new URL(url);
-		const text = u.pathname.split("/").filter(Boolean).map((s) => decodeURIComponent(s).toLowerCase()).map((s) => s.replace(HTML_EXT_RE, "")).map((s) => s.replace(SEP_RE, " ")).filter((s) => s.length > 0 && !DIGITS_ONLY_RE.test(s)).join(" ");
-		if (text.length > 0) return text;
-		return u.hostname.replace(WWW_RE, "").replace(DOT_RE, " ");
-	} catch {
-		return url.replace(ORIGIN_RE, "").replace(SEP_RE, " ");
-	}
-}
-function cosineNormalized(a, b) {
-	let dot = 0;
-	const n = a.length;
-	for (let i = 0; i < n; i++) dot += a[i] * b[i];
-	return dot;
-}
-function notify(onProgress, progress) {
-	onProgress?.(progress);
-}
 function openDb() {
 	if (dbPromise != null) return dbPromise;
 	dbPromise = new Promise((resolve, reject) => {
@@ -63,7 +17,7 @@ function openDb() {
 	return dbPromise;
 }
 function cacheKey(role, text) {
-	return `${MODEL_ID}|${role}|${text}`;
+	return `${CONTENT_GAP_MODEL_ID}|${role}|${text}`;
 }
 async function cacheGetMany(role, texts) {
 	const db = await openDb().catch(() => null);
@@ -119,7 +73,25 @@ async function embedRawBatch(extractor, texts, onProgress, batchSize = 32) {
 	}
 	return result;
 }
-async function embedCached(extractor, role, texts, transform, onProgress) {
+async function selectContentGapDevice(requested) {
+	let chosenDevice = "wasm";
+	if (requested === "webgpu" || requested == null) {
+		const gpu = globalThis.navigator?.gpu;
+		if (gpu != null) {
+			if (await gpu.requestAdapter().catch(() => null) != null) chosenDevice = "webgpu";
+		}
+	}
+	return chosenDevice;
+}
+async function loadContentGapExtractor(device) {
+	const { pipeline, env } = await import("@huggingface/transformers");
+	env.useBrowserCache = true;
+	return await pipeline("feature-extraction", CONTENT_GAP_MODEL_ID, {
+		device,
+		dtype: "fp32"
+	});
+}
+async function embedContentGapTexts(extractor, role, texts, transform, onProgress) {
 	const cached = await cacheGetMany(role, texts);
 	const vectors = Array.from({ length: texts.length });
 	const missIdx = [];
@@ -153,25 +125,7 @@ async function embedCached(extractor, role, texts, transform, onProgress) {
 		misses
 	};
 }
-async function selectDevice(requested) {
-	let chosenDevice = "wasm";
-	if (requested === "webgpu" || requested == null) {
-		const gpu = globalThis.navigator?.gpu;
-		if (gpu != null) {
-			if (await gpu.requestAdapter().catch(() => null) != null) chosenDevice = "webgpu";
-		}
-	}
-	return chosenDevice;
-}
-async function loadExtractor(device) {
-	const { pipeline, env } = await import("@huggingface/transformers");
-	env.useBrowserCache = true;
-	return await pipeline("feature-extraction", MODEL_ID, {
-		device,
-		dtype: "fp32"
-	});
-}
-async function fetchContentGapInputs(executeSql, options) {
+async function fetchContentGapInputs(executeSql, options, normalizeUrl) {
 	const t1 = performance.now();
 	const queryRows = await executeSql(`
     WITH query_totals AS (
@@ -229,6 +183,52 @@ async function fetchContentGapInputs(executeSql, options) {
 		sqlMs
 	};
 }
+var ContentGapSourceUnsupportedError = class extends Error {
+	constructor(kind) {
+		super(`content-gap requires a source with executeSql (got '${kind}'); use createBrowserQuerySource or createSqliteQuerySource`);
+		this.name = "ContentGapSourceUnsupportedError";
+	}
+};
+const ORIGIN_RE = /^https?:\/\/[^/]+/;
+const HTML_EXT_RE = /\.(html?|php|aspx?)$/i;
+const SEP_RE = /[-_]+/g;
+const DIGITS_ONLY_RE = /^\d+$/;
+const WWW_RE = /^www\./;
+const DOT_RE = /\./g;
+const HASH_RE = /#.*$/;
+const QUERY_RE = /\?.*$/;
+const TRAIL_SLASH_RE = /(?<=.)\/$/;
+function normalizeUrl(u) {
+	try {
+		const url = new URL(u);
+		url.hash = "";
+		url.search = "";
+		let path = url.pathname;
+		if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
+		return `${url.origin}${path}`;
+	} catch {
+		return u.replace(HASH_RE, "").replace(QUERY_RE, "").replace(TRAIL_SLASH_RE, "");
+	}
+}
+function deriveUrlText(url) {
+	try {
+		const u = new URL(url);
+		const text = u.pathname.split("/").filter(Boolean).map((s) => decodeURIComponent(s).toLowerCase()).map((s) => s.replace(HTML_EXT_RE, "")).map((s) => s.replace(SEP_RE, " ")).filter((s) => s.length > 0 && !DIGITS_ONLY_RE.test(s)).join(" ");
+		if (text.length > 0) return text;
+		return u.hostname.replace(WWW_RE, "").replace(DOT_RE, " ");
+	} catch {
+		return url.replace(ORIGIN_RE, "").replace(SEP_RE, " ");
+	}
+}
+function cosineNormalized(a, b) {
+	let dot = 0;
+	const n = a.length;
+	for (let i = 0; i < n; i++) dot += a[i] * b[i];
+	return dot;
+}
+function notify(onProgress, progress) {
+	onProgress?.(progress);
+}
 function rankContentGaps(queries, urls, queryEmbeddings, urlEmbeddings, minDivergence) {
 	const urlIndex = /* @__PURE__ */ new Map();
 	for (let i = 0; i < urls.length; i++) urlIndex.set(urls[i], i);
@@ -275,12 +275,12 @@ async function analyzeContentGap(source, opts = {}) {
 		message: "Checking device..."
 	});
 	const t0 = performance.now();
-	const chosenDevice = await selectDevice(device);
+	const chosenDevice = await selectContentGapDevice(device);
 	notify(onProgress, {
 		phase: "loading-model",
-		message: `Loading ${MODEL_ID} on ${chosenDevice} (~110MB, cached after first run)...`
+		message: `Loading ${CONTENT_GAP_MODEL_ID} on ${chosenDevice} (~110MB, cached after first run)...`
 	});
-	const extractor = await loadExtractor(chosenDevice);
+	const extractor = await loadContentGapExtractor(chosenDevice);
 	const modelMs = performance.now() - t0;
 	notify(onProgress, {
 		phase: "fetching-data",
@@ -291,7 +291,7 @@ async function analyzeContentGap(source, opts = {}) {
 		maxQueries,
 		maxUrls,
 		minImpressions
-	});
+	}, normalizeUrl);
 	if (queries.length === 0 || urls.length === 0) {
 		notify(onProgress, {
 			phase: "done",
@@ -309,7 +309,7 @@ async function analyzeContentGap(source, opts = {}) {
 				cacheHits: 0,
 				totalInputs: 0,
 				device: chosenDevice,
-				modelId: MODEL_ID
+				modelId: CONTENT_GAP_MODEL_ID
 			}
 		};
 	}
@@ -324,7 +324,7 @@ async function analyzeContentGap(source, opts = {}) {
 		sqlMs
 	});
 	const t2 = performance.now();
-	const queryEmbed = await embedCached(extractor, "query", queryTexts, (t) => QUERY_PREFIX + t, (done, total) => {
+	const queryEmbed = await embedContentGapTexts(extractor, "query", queryTexts, (t) => CONTENT_GAP_QUERY_PREFIX + t, (done, total) => {
 		notify(onProgress, {
 			phase: "embedding-queries",
 			message: `Embedding ${queryTexts.length} queries on ${chosenDevice}...`,
@@ -342,7 +342,7 @@ async function analyzeContentGap(source, opts = {}) {
 		modelMs,
 		sqlMs
 	});
-	const urlEmbed = await embedCached(extractor, "passage", urlTexts, (t) => t, (done, total) => {
+	const urlEmbed = await embedContentGapTexts(extractor, "passage", urlTexts, (t) => t, (done, total) => {
 		notify(onProgress, {
 			phase: "embedding-urls",
 			message: `Embedding ${urls.length} URLs...`,
@@ -384,7 +384,7 @@ async function analyzeContentGap(source, opts = {}) {
 			cacheHits: totalHits,
 			totalInputs,
 			device: chosenDevice,
-			modelId: MODEL_ID
+			modelId: CONTENT_GAP_MODEL_ID
 		}
 	};
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/analysis",
   "type": "module",
-  "version": "0.33.0",
+  "version": "0.33.4",
   "description": "GSC analyzers — striking-distance, opportunity, movers, decay, brand, clustering, concentration, seasonality. Pure row-based + DuckDB-native.",
   "author": {
     "name": "Harlan Wilton",
@@ -76,9 +76,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "pluralize": "^8.0.0",
-    "@gscdump/engine": "0.33.0",
-    "@gscdump/engine-gsc-api": "0.33.0",
-    "gscdump": "0.33.0"
+    "@gscdump/engine": "0.33.4",
+    "@gscdump/engine-gsc-api": "0.33.4",
+    "gscdump": "0.33.4"
   },
   "devDependencies": {
     "vitest": "^4.1.9"