@gscdump/cli 0.38.1 → 0.39.0

package/dist/_chunks/dump.mjs

@@ -1,11 +1,10 @@
-import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, toCSV } from "./utils.mjs";
+import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, runWithConcurrency, toCSV } from "./utils.mjs";
 import { allTables, createCommandContext } from "./context.mjs";
 import { readParquetRows } from "./native-duckdb.mjs";
 import process from "node:process";
 import { defineCommand } from "citty";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { Buffer } from "node:buffer";
 const DEFAULT_OUT = "./gscdump-export";
 const FORMATS = [
 	"parquet",
@@ -72,32 +71,48 @@ const dumpCommand = defineCommand({
 		});
 		const store = ctx.store;
 		const outDir = path.resolve(String(args.out));
-		const targets = args["all-sites"] ? await listSitesWithData(store) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
+		let preloadedEntries = args["all-sites"] ? await store.engine.listLive({
+			userId: store.userId,
+			...searchType !== void 0 ? { searchType } : {}
+		}) : void 0;
+		const targets = args["all-sites"] ? [...new Set(preloadedEntries.flatMap((entry) => entry.siteId ? [entry.siteId] : []))].map((siteId) => ({
+			site: siteId,
+			siteId
+		})) : await ctx.resolveSite(args.site ? String(args.site) : void 0).then((site) => [{
+			site,
+			siteId: store.siteIdFor(site)
+		}]);
 		if (targets.length === 0) {
 			logger.warn("No sites with local data. Run `gscdump sync` first.");
 			process.exit(0);
 		}
-		if (args.compact) for (const siteUrl of targets) await compactClosedMonths(store, siteUrl, quiet);
+		if (args.compact) {
+			for (const target of targets) await compactClosedMonths(store, target.siteId, quiet);
+			if (preloadedEntries) preloadedEntries = await store.engine.listLive({
+				userId: store.userId,
+				...searchType !== void 0 ? { searchType } : {}
+			});
+		}
 		const summary = [];
-		for (const siteUrl of targets) {
-			const entries = (await listLiveEntries(store, siteUrl, searchType)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
+		for (const target of targets) {
+			const entries = (preloadedEntries ? preloadedEntries.filter((entry) => entry.siteId === target.siteId) : await listLiveEntries(store, target.siteId, searchType)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
 			if (entries.length === 0) {
-				if (!quiet) logger.warn(`No data for ${siteUrl}; skipping`);
+				if (!quiet) logger.warn(`No data for ${target.site}; skipping`);
 				continue;
 			}
 			if (format === "parquet") {
 				const written = await dumpParquet(store, entries, outDir);
 				summary.push({
-					site: siteUrl,
+					site: target.site,
 					files: written,
 					rows: 0,
 					format,
 					outPath: outDir
 				});
 			} else {
-				const written = await dumpRowFormat(store, entries, outDir, siteUrl, format);
+				const written = await dumpRowFormat(store, entries, outDir, target.site, format);
 				summary.push({
-					site: siteUrl,
+					site: target.site,
 					files: written.files,
 					rows: written.rows,
 					format,
@@ -118,36 +133,33 @@ const dumpCommand = defineCommand({
 		}
 	}
 });
-async function listSitesWithData(store) {
-	const siteIds = /* @__PURE__ */ new Set();
-	for (const table of allTables()) {
-		const entries = await store.engine.listLive({
-			userId: store.userId,
-			table
-		});
-		for (const e of entries) if (e.siteId) siteIds.add(e.siteId);
-	}
-	return Array.from(siteIds);
-}
-async function listLiveEntries(store, siteUrl, searchType) {
-	const siteId = store.siteIdFor(siteUrl);
-	return (await Promise.all(allTables().map((table) => store.engine.listLive({
+async function listLiveEntries(store, siteId, searchType) {
+	return store.engine.listLive({
 		userId: store.userId,
 		siteId,
-		table,
 		...searchType !== void 0 ? { searchType } : {}
-	})))).flat();
+	});
 }
 async function dumpParquet(store, entries, outDir) {
 	await fs.mkdir(outDir, { recursive: true });
+	const readyDirectories = /* @__PURE__ */ new Map();
+	async function ensureDirectory(dir) {
+		let ready = readyDirectories.get(dir);
+		if (!ready) {
+			ready = fs.mkdir(dir, { recursive: true }).then(() => void 0);
+			readyDirectories.set(dir, ready);
+			ready.catch(() => readyDirectories.delete(dir));
+		}
+		await ready;
+	}
 	let copied = 0;
-	for (const entry of entries) {
+	await runWithConcurrency(entries, 8, async (entry) => {
 		const bytes = await store.engine.readObject(entry.objectKey);
 		const target = path.join(outDir, entry.objectKey);
-		await fs.mkdir(path.dirname(target), { recursive: true });
-		await fs.writeFile(target, Buffer.from(bytes));
+		await ensureDirectory(path.dirname(target));
+		await fs.writeFile(target, bytes);
 		copied++;
-	}
+	});
 	return copied;
 }
 async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
@@ -179,8 +191,7 @@ async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
 		rows: totalRows
 	};
 }
-async function compactClosedMonths(store, siteUrl, quiet) {
-	const siteId = store.siteIdFor(siteUrl);
+async function compactClosedMonths(store, siteId, quiet) {
 	for (const table of allTables()) {
 		if (!quiet) logger.info(`Compacting ${table} (raw→d7→d30→d90)`);
 		await store.engine.compactTiered({

package/dist/_chunks/store.mjs

@@ -49,14 +49,14 @@ const compactCommand = defineCommand({
 		if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
 		if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
 		if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
+		const liveEntries = await store.engine.listLive({
+			userId: store.userId,
+			siteId
+		});
 		if (dryRun) {
 			const report = [];
 			for (const table of allTables()) {
-				const bySite = groupBySite(await store.engine.listLive({
-					userId: store.userId,
-					siteId,
-					table
-				}));
+				const bySite = groupBySite(liveEntries.filter((entry) => entry.table === table));
 				for (const [s, group] of bySite) report.push({
 					table,
 					siteId: s,
@@ -79,11 +79,7 @@ const compactCommand = defineCommand({
 		}
 		const summary = [];
 		for (const table of allTables()) {
-			const entries = await store.engine.listLive({
-				userId: store.userId,
-				siteId,
-				table
-			});
+			const entries = liveEntries.filter((entry) => entry.table === table);
 			const siteIds = new Set(entries.map((e) => e.siteId));
 			for (const targetSite of siteIds) {
 				logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
@@ -138,16 +134,16 @@ function countByTier(entries) {
 }
 async function exportToDuckDB(opts) {
 	const outPath = path.resolve(opts.outPath);
+	const entries = await opts.engine.listLive({
+		userId: opts.userId,
+		siteId: opts.siteId
+	});
 	const inputs = [];
 	for (const table of allTables()) {
-		const entries = await opts.engine.listLive({
-			userId: opts.userId,
-			siteId: opts.siteId,
-			table
-		});
-		if (entries.length > 0) inputs.push({
+		const tableEntries = entries.filter((entry) => entry.table === table);
+		if (tableEntries.length > 0) inputs.push({
 			table,
-			filePaths: entries.map((entry) => path.join(opts.dataDir, entry.objectKey))
+			filePaths: tableEntries.map((entry) => path.join(opts.dataDir, entry.objectKey))
 		});
 	}
 	const tables = await materializeParquetTables(outPath, inputs, opts.force);
@@ -237,20 +233,17 @@ const gcCommand = defineCommand({
 		if (args["dry-run"]) {
 			const cutoff = Date.now() - graceMs;
 			const candidates = [];
-			for (const table of allTables()) {
-				const all = await store.engine.listAll({
-					userId: store.userId,
-					siteId,
-					table
-				});
-				for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
-					table,
-					siteId: e.siteId,
-					partition: e.partition,
-					retiredAt: e.retiredAt,
-					objectKey: e.objectKey
-				});
-			}
+			const all = await store.engine.listAll({
+				userId: store.userId,
+				siteId
+			});
+			for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
+				table: e.table,
+				siteId: e.siteId,
+				partition: e.partition,
+				retiredAt: e.retiredAt,
+				objectKey: e.objectKey
+			});
 			if (json) {
 				console.log(JSON.stringify({
 					graceHours: Number(args["grace-hours"]),
@@ -337,11 +330,8 @@ const rollupsCommand = defineCommand({
 			const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
 			const allSiteIds = /* @__PURE__ */ new Set();
 			if (explicitSiteId) allSiteIds.add(explicitSiteId);
-			else for (const table of allTables()) {
-				const entries = await store.engine.listLive({
-					userId: store.userId,
-					table
-				});
+			else {
+				const entries = await store.engine.listLive({ userId: store.userId });
 				for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
 			}
 			if (allSiteIds.size === 0) {
@@ -423,9 +413,10 @@ const statsCommand = defineCommand({
 	async run({ args }) {
 		const { json } = applyOutputMode(args);
 		const store = (await createCommandContext({ needsStore: true })).store;
+		const allEntries = await store.engine.listAll({ userId: store.userId });
 		let siteId;
 		if (args.site) {
-			const known = await listKnownSiteIds(store);
+			const known = new Set(allEntries.filter((entry) => entry.retiredAt === void 0 && entry.siteId !== void 0).map((entry) => entry.siteId));
 			const candidate = store.siteIdFor(args.site);
 			if (!known.has(candidate)) {
 				logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
@@ -433,26 +424,22 @@ const statsCommand = defineCommand({
 			}
 			siteId = candidate;
 		}
-		const perTable = await Promise.all(allTables().map(async (table) => {
-			const all = await store.engine.listAll({
-				userId: store.userId,
-				siteId,
-				table
-			});
+		const selectedEntries = siteId === void 0 ? allEntries : allEntries.filter((entry) => entry.siteId === siteId);
+		const perTable = allTables().map((table) => {
+			const all = selectedEntries.filter((entry) => entry.table === table);
 			return {
 				table,
 				live: all.filter((e) => e.retiredAt === void 0),
 				retired: all.filter((e) => e.retiredAt !== void 0)
 			};
-		}));
-		const watermarks = await store.engine.getWatermarks({
+		});
+		const [watermarks, disk] = await Promise.all([store.engine.getWatermarks({
 			userId: store.userId,
 			siteId
-		});
-		const disk = await filesystemStats(store.dataDir).catch(() => ({
+		}), filesystemStats(store.dataDir).catch(() => ({
 			files: 0,
 			bytes: 0
-		}));
+		}))]);
 		if (json) {
 			const payload = {
 				dataDir: store.dataDir,
@@ -504,17 +491,6 @@ const statsCommand = defineCommand({
 		console.log();
 	}
 });
-async function listKnownSiteIds(store) {
-	const ids = /* @__PURE__ */ new Set();
-	for (const table of allTables()) {
-		const entries = await store.engine.listLive({
-			userId: store.userId,
-			table
-		});
-		for (const e of entries) if (e.siteId) ids.add(e.siteId);
-	}
-	return ids;
-}
 function sortWatermarks(ws) {
 	return [...ws].sort((a, b) => {
 		if (a.table !== b.table) return a.table.localeCompare(b.table);

package/dist/_chunks/sync.mjs

@@ -269,15 +269,13 @@ const syncCommand = defineCommand({
 		const store = ctx.store;
 		if (args["retry-failed"]) {
 			const failedSet = /* @__PURE__ */ new Set();
-			for (const table of tables) for (const type of types) {
-				const states = await store.engine.getSyncStates({
-					userId: store.userId,
-					siteId,
-					table,
-					searchType: type
-				});
-				for (const s of states) if (s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
-			}
+			const selectedTables = new Set(tables);
+			const selectedTypes = new Set(types);
+			const states = await store.engine.getSyncStates({
+				userId: store.userId,
+				siteId
+			});
+			for (const s of states) if (selectedTables.has(s.table) && selectedTypes.has(s.searchType ?? "web") && s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
 			dates = dates.filter((d) => failedSet.has(d));
 			if (dates.length === 0) {
 				logger.success("No failed dates in range — nothing to retry.");

package/dist/_chunks/utils.mjs

@@ -4,7 +4,7 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import { Buffer } from "node:buffer";
 import { SearchTypes } from "gscdump/query";
-var version = "0.38.1";
+var version = "0.39.0";
 const ALL_SEARCH_TYPES = Object.values(SearchTypes);
 const VERSION = version;
 function noSubcommandSelected(parent, subNames) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cli",
   "type": "module",
-  "version": "0.38.1",
+  "version": "0.39.0",
   "description": "CLI for Google Search Console - dump, query, and run MCP server",
   "author": {
     "name": "Harlan Wilton",
@@ -52,10 +52,10 @@
     "ofetch": "^1.5.1",
     "open": "^11.0.0",
     "zod": "^4.4.3",
-    "@gscdump/analysis": "0.38.1",
-    "@gscdump/engine": "0.38.1",
-    "@gscdump/engine-gsc-api": "0.38.1",
-    "gscdump": "0.38.1"
+    "@gscdump/analysis": "0.39.0",
+    "@gscdump/engine": "0.39.0",
+    "@gscdump/engine-gsc-api": "0.39.0",
+    "gscdump": "0.39.0"
   },
   "devDependencies": {
     "vitest": "^4.1.10"