@gscdump/cli 1.4.0 → 1.4.2

package/dist/_chunks/store.mjs

@@ -1,628 +0,0 @@
-import { OUTPUT_ARGS, applyOutputMode, displayPath, formatAge, logger, noSubcommandSelected } from "./utils.mjs";
-import { allTables, createCommandContext } from "./context.mjs";
-import { materializeParquetTables } from "./native-duckdb.mjs";
-import process from "node:process";
-import { defineCommand } from "citty";
-import path from "node:path";
-import { confirm, isCancel } from "@clack/prompts";
-import { INTENT_CLASSIFIER_VERSION, NORMALIZER_VERSION, classifyQueryIntent, encodeIntent, normalizeQuery } from "@gscdump/analysis";
-import { buildQueryDimRecords, createQueryDimStore } from "@gscdump/engine/entities";
-import { inferLegacyTier } from "@gscdump/engine";
-import { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/engine/rollups";
-import { filesystemStats } from "@gscdump/engine/filesystem";
-const compactCommand = defineCommand({
-	meta: {
-		name: "compact",
-		description: "Run tiered compaction (raw→d7 at 7d, d7→d30 at 30d, d30→d90 at 90d)"
-	},
-	args: {
-		"site": {
-			type: "string",
-			alias: "s",
-			description: "Restrict to a single site (default: all sites with local data)"
-		},
-		"raw-days": {
-			type: "string",
-			description: "Override raw→d7 age threshold in days (default: 7)"
-		},
-		"d7-days": {
-			type: "string",
-			description: "Override d7→d30 age threshold in days (default: 30)"
-		},
-		"d30-days": {
-			type: "string",
-			description: "Override d30→d90 age threshold in days (default: 90)"
-		},
-		"dry-run": {
-			type: "boolean",
-			default: false,
-			description: "Report tier counts per (table, site) without compacting"
-		},
-		...OUTPUT_ARGS
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const store = (await createCommandContext({ needsStore: true })).store;
-		const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
-		const dryRun = Boolean(args["dry-run"]);
-		const thresholds = {};
-		if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
-		if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
-		if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
-		const liveEntries = await store.engine.listLive({
-			userId: store.userId,
-			siteId
-		});
-		if (dryRun) {
-			const report = [];
-			for (const table of allTables()) {
-				const bySite = groupBySite(liveEntries.filter((entry) => entry.table === table));
-				for (const [s, group] of bySite) report.push({
-					table,
-					siteId: s,
-					...countByTier(group)
-				});
-			}
-			if (json) {
-				console.log(JSON.stringify({
-					thresholds,
-					plan: report
-				}, null, 2));
-				return;
-			}
-			console.log();
-			console.log(`  table                site                 raw    d7   d30   d90`);
-			for (const r of report) console.log(`  ${r.table.padEnd(20)} ${(r.siteId ?? "-").padEnd(20)} ${String(r.raw).padStart(4)}  ${String(r.d7).padStart(4)}  ${String(r.d30).padStart(4)}  ${String(r.d90).padStart(4)}`);
-			console.log();
-			logger.info(`compact --dry-run: ${report.length} (table, site) pair(s) — pass without --dry-run to apply`);
-			return;
-		}
-		const summary = [];
-		for (const table of allTables()) {
-			const entries = liveEntries.filter((entry) => entry.table === table);
-			const siteIds = new Set(entries.map((e) => e.siteId));
-			for (const targetSite of siteIds) {
-				logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
-				await store.engine.compactTiered({
-					userId: store.userId,
-					siteId: targetSite,
-					table
-				}, thresholds);
-				summary.push({
-					table,
-					siteId: targetSite
-				});
-			}
-		}
-		if (json) {
-			console.log(JSON.stringify({
-				thresholds,
-				compacted: summary
-			}, null, 2));
-			return;
-		}
-		logger.success(`compact: done`);
-	}
-});
-function groupBySite(entries) {
-	const m = /* @__PURE__ */ new Map();
-	for (const e of entries) {
-		const arr = m.get(e.siteId) ?? [];
-		arr.push(e);
-		m.set(e.siteId, arr);
-	}
-	return m;
-}
-function countByTier(entries) {
-	let raw = 0;
-	let d7 = 0;
-	let d30 = 0;
-	let d90 = 0;
-	for (const e of entries) {
-		const tier = e.tier ?? inferLegacyTier(e) ?? "raw";
-		if (tier === "raw") raw++;
-		else if (tier === "d7") d7++;
-		else if (tier === "d30") d30++;
-		else if (tier === "d90") d90++;
-	}
-	return {
-		raw,
-		d7,
-		d30,
-		d90
-	};
-}
-async function exportToDuckDB(opts) {
-	const outPath = path.resolve(opts.outPath);
-	const entries = await opts.engine.listLive({
-		userId: opts.userId,
-		siteId: opts.siteId
-	});
-	const inputs = [];
-	for (const table of allTables()) {
-		const tableEntries = entries.filter((entry) => entry.table === table);
-		if (tableEntries.length > 0) inputs.push({
-			table,
-			filePaths: tableEntries.map((entry) => path.join(opts.dataDir, entry.objectKey))
-		});
-	}
-	const tables = await materializeParquetTables(outPath, inputs, opts.force);
-	return {
-		outPath,
-		tables,
-		totalRows: tables.reduce((acc, t) => acc + t.rows, 0)
-	};
-}
-const exportCommand = defineCommand({
-	meta: {
-		name: "export",
-		description: "Pack live Parquet partitions into a single .duckdb file for portable distribution (browser attach, CDN serving, etc.)"
-	},
-	args: {
-		out: {
-			type: "string",
-			required: true,
-			description: "Output path for the .duckdb file"
-		},
-		site: {
-			type: "string",
-			description: "Limit export to a single site URL (omit to include all)"
-		},
-		force: {
-			type: "boolean",
-			default: false,
-			description: "Overwrite the output file if it already exists"
-		},
-		...OUTPUT_ARGS
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const store = (await createCommandContext({ needsStore: true })).store;
-		const siteId = args.site ? store.siteIdFor(args.site) : void 0;
-		const result = await exportToDuckDB({
-			engine: store.engine,
-			dataDir: store.dataDir,
-			userId: store.userId,
-			siteId,
-			outPath: args.out,
-			force: args.force
-		});
-		if (json) {
-			console.log(JSON.stringify(result, null, 2));
-			return;
-		}
-		if (result.tables.length === 0) {
-			console.log(`\n  No data to export. Run \`gscdump sync\` first.`);
-			return;
-		}
-		for (const t of result.tables) console.log(`  ${t.table.padEnd(15)} ${String(t.files).padStart(4)} parquet → ${t.table}  (${t.rows.toLocaleString()} rows)`);
-		console.log(`\n  Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${displayPath(result.outPath)}`);
-		console.log(`\n  Attach from DuckDB:     \x1B[36mATTACH '${result.outPath}' AS gsc (READ_ONLY); SELECT * FROM gsc.pages LIMIT 10;\x1B[0m`);
-		console.log(`  Attach in a browser:    use DuckDB-WASM registerFileBuffer + \x1B[36mATTACH 'gsc.duckdb' AS gsc (READ_ONLY)\x1B[0m`);
-	}
-});
-const DEFAULT_GRACE_HOURS = 24;
-const gcCommand = defineCommand({
-	meta: {
-		name: "gc",
-		description: "Delete orphaned object-store files not referenced by any manifest entry"
-	},
-	args: {
-		"grace-hours": {
-			type: "string",
-			default: String(DEFAULT_GRACE_HOURS),
-			description: `Spare orphans younger than this (default: ${DEFAULT_GRACE_HOURS}h)`
-		},
-		"site": {
-			type: "string",
-			alias: "s",
-			description: "Restrict to a single site (default: all sites)"
-		},
-		"dry-run": {
-			type: "boolean",
-			default: false,
-			description: "List retired manifest entries past the grace window without deleting"
-		},
-		...OUTPUT_ARGS
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const store = (await createCommandContext({ needsStore: true })).store;
-		const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
-		const graceMs = Number(args["grace-hours"]) * 36e5;
-		if (args["dry-run"]) {
-			const cutoff = Date.now() - graceMs;
-			const candidates = [];
-			const all = await store.engine.listAll({
-				userId: store.userId,
-				siteId
-			});
-			for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
-				table: e.table,
-				siteId: e.siteId,
-				partition: e.partition,
-				retiredAt: e.retiredAt,
-				objectKey: e.objectKey
-			});
-			if (json) {
-				console.log(JSON.stringify({
-					graceHours: Number(args["grace-hours"]),
-					candidates
-				}, null, 2));
-				return;
-			}
-			console.log();
-			for (const c of candidates) console.log(`  ${c.objectKey}  \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
-			console.log();
-			logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
-			return;
-		}
-		const result = await store.engine.gcOrphans({
-			userId: store.userId,
-			siteId
-		}, graceMs);
-		if (json) {
-			console.log(JSON.stringify({
-				graceHours: Number(args["grace-hours"]),
-				deleted: result.deleted
-			}, null, 2));
-			return;
-		}
-		logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
-	}
-});
-async function buildSiteQueryDim(store, siteId) {
-	const ctx = {
-		userId: store.userId,
-		siteId
-	};
-	const entries = await store.engine.listLive({
-		userId: ctx.userId,
-		siteId,
-		table: "queries"
-	});
-	if (entries.length === 0) return 0;
-	const { rows } = await store.engine.runSQL({
-		ctx,
-		table: "queries",
-		fileSets: { FILES: {
-			table: "queries",
-			partitions: entries.map((e) => e.partition)
-		} },
-		sql: `SELECT DISTINCT query FROM read_parquet({{FILES}}, union_by_name = true) WHERE query IS NOT NULL`
-	});
-	const records = buildQueryDimRecords(rows.map((r) => String(r.query)), {
-		normalizeQuery,
-		normalizerVersion: NORMALIZER_VERSION,
-		classifyIntentCode: (q) => encodeIntent(classifyQueryIntent(q)),
-		intentVersion: INTENT_CLASSIFIER_VERSION
-	});
-	await createQueryDimStore({ dataSource: store.dataSource }).write(ctx, records, Date.now());
-	return records.length;
-}
-const rollupsCommand = defineCommand({
-	meta: {
-		name: "rollups",
-		description: "Manage post-sync rollups"
-	},
-	subCommands: { rebuild: defineCommand({
-		meta: {
-			name: "rebuild",
-			description: "Rebuild post-sync rollups (daily totals, weekly totals, top-N tables) for a site"
-		},
-		args: {
-			...OUTPUT_ARGS,
-			"site": {
-				type: "string",
-				alias: "s",
-				description: "Restrict to a single site (default: all sites with local data)"
-			},
-			"with-canonical": {
-				type: "boolean",
-				description: "Also build the opt-in canonical-primary rollups (query_canonical_variants, query_canonical_daily)",
-				default: false
-			}
-		},
-		async run({ args }) {
-			const { json } = applyOutputMode(args);
-			const defs = args["with-canonical"] ? [...DEFAULT_ROLLUPS, ...CANONICAL_ROLLUPS] : DEFAULT_ROLLUPS;
-			const store = (await createCommandContext({ needsStore: true })).store;
-			const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
-			const allSiteIds = /* @__PURE__ */ new Set();
-			if (explicitSiteId) allSiteIds.add(explicitSiteId);
-			else {
-				const entries = await store.engine.listLive({ userId: store.userId });
-				for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
-			}
-			if (allSiteIds.size === 0) {
-				if (json) console.log(JSON.stringify({
-					sites: [],
-					totalBytes: 0
-				}, null, 2));
-				else logger.warn("No sites with local data. Run `gscdump sync` first.");
-				return;
-			}
-			const summary = [];
-			let totalBytes = 0;
-			for (const siteId of allSiteIds) {
-				if (args["with-canonical"]) {
-					const dimRows = await buildSiteQueryDim(store, siteId);
-					if (!json) logger.info(`Built query dimension for [${siteId}] (${dimRows} distinct queries, normalizer v${NORMALIZER_VERSION})`);
-				}
-				logger.info(`Rebuilding rollups for [${siteId}] (${defs.length} rollups)`);
-				const results = await rebuildRollups({
-					engine: {
-						runSQL: (opts) => store.engine.runSQL(opts),
-						listPartitions: async ({ ctx, table, searchType }) => {
-							return (await store.engine.listLive({
-								userId: ctx.userId,
-								...ctx.siteId !== void 0 ? { siteId: ctx.siteId } : {},
-								table,
-								...searchType !== void 0 ? { searchType } : {}
-							})).map((e) => ({
-								partition: e.partition,
-								bytes: e.bytes
-							}));
-						}
-					},
-					dataSource: store.dataSource,
-					ctx: {
-						userId: store.userId,
-						siteId
-					},
-					defs
-				});
-				const site = {
-					siteId,
-					rollups: []
-				};
-				for (const r of results) {
-					totalBytes += r.bytes;
-					site.rollups.push({
-						id: r.id,
-						bytes: r.bytes,
-						objectKey: r.objectKey
-					});
-					if (!json) console.log(`  ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB  ${r.objectKey}`);
-				}
-				summary.push(site);
-			}
-			if (json) {
-				console.log(JSON.stringify({
-					sites: summary,
-					totalBytes
-				}, null, 2));
-				return;
-			}
-			logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
-		}
-	}) }
-});
-const statsCommand = defineCommand({
-	meta: {
-		name: "stats",
-		description: "Show row/byte counts per table and on-disk footprint"
-	},
-	args: {
-		...OUTPUT_ARGS,
-		site: {
-			type: "string",
-			description: "Limit to one site URL (sc-domain:example.com, https://example.com/, ...)"
-		}
-	},
-	async run({ args }) {
-		const { json } = applyOutputMode(args);
-		const store = (await createCommandContext({ needsStore: true })).store;
-		const allEntries = await store.engine.listAll({ userId: store.userId });
-		let siteId;
-		if (args.site) {
-			const known = new Set(allEntries.filter((entry) => entry.retiredAt === void 0 && entry.siteId !== void 0).map((entry) => entry.siteId));
-			const candidate = store.siteIdFor(args.site);
-			if (!known.has(candidate)) {
-				logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
-				process.exit(1);
-			}
-			siteId = candidate;
-		}
-		const selectedEntries = siteId === void 0 ? allEntries : allEntries.filter((entry) => entry.siteId === siteId);
-		const perTable = allTables().map((table) => {
-			const all = selectedEntries.filter((entry) => entry.table === table);
-			return {
-				table,
-				live: all.filter((e) => e.retiredAt === void 0),
-				retired: all.filter((e) => e.retiredAt !== void 0)
-			};
-		});
-		const [watermarks, disk] = await Promise.all([store.engine.getWatermarks({
-			userId: store.userId,
-			siteId
-		}), filesystemStats(store.dataDir).catch(() => ({
-			files: 0,
-			bytes: 0
-		}))]);
-		if (json) {
-			const payload = {
-				dataDir: store.dataDir,
-				disk,
-				tables: perTable.map(({ table, live, retired }) => ({
-					table,
-					liveFiles: live.length,
-					liveRows: sumRows(live),
-					liveBytes: sumBytes(live),
-					retiredFiles: retired.length,
-					retiredBytes: sumBytes(retired),
-					watermarks: watermarks.filter((w) => w.table === table).map((w) => ({
-						siteId: w.siteId ?? null,
-						newestDateSynced: w.newestDateSynced,
-						oldestDateSynced: w.oldestDateSynced,
-						lastSyncAt: w.lastSyncAt
-					}))
-				}))
-			};
-			console.log(JSON.stringify(payload, null, 2));
-			return;
-		}
-		console.log();
-		console.log(`  \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
-		console.log(`  \x1B[90mDisk: ${disk.files} file(s), ${formatBytes(disk.bytes)}\x1B[0m`);
-		console.log();
-		const totalRows = perTable.reduce((acc, t) => acc + sumRows(t.live), 0);
-		const totalBytes = perTable.reduce((acc, t) => acc + sumBytes(t.live), 0);
-		const totalFiles = perTable.reduce((acc, t) => acc + t.live.length, 0);
-		const totalRetiredFiles = perTable.reduce((acc, t) => acc + t.retired.length, 0);
-		const totalRetiredBytes = perTable.reduce((acc, t) => acc + sumBytes(t.retired), 0);
-		for (const { table, live, retired } of perTable) {
-			const rows = sumRows(live).toLocaleString();
-			const bytes = formatBytes(sumBytes(live));
-			const retiredSuffix = retired.length > 0 ? ` \x1B[90m(+${retired.length} retired, ${formatBytes(sumBytes(retired))})\x1B[0m` : "";
-			console.log(`  ${table.padEnd(15)} \x1B[36m${String(live.length).padStart(4)}\x1B[0m files, ${rows.padStart(10)} rows, ${bytes}${retiredSuffix}`);
-		}
-		console.log();
-		console.log(`  \x1B[1mTotal:\x1B[0m ${totalFiles} files, ${totalRows.toLocaleString()} rows, ${formatBytes(totalBytes)} live`);
-		if (totalRetiredFiles > 0) console.log(`  \x1B[90mRetired: ${totalRetiredFiles} files, ${formatBytes(totalRetiredBytes)} awaiting GC\x1B[0m`);
-		if (watermarks.length > 0) {
-			console.log();
-			console.log(`  \x1B[1mSync watermarks:\x1B[0m`);
-			for (const w of sortWatermarks(watermarks)) {
-				const scope = w.siteId ? `${w.table}@${w.siteId}` : w.table;
-				console.log(`  ${scope.padEnd(24)} \x1B[36m${w.oldestDateSynced}\x1B[0m → \x1B[36m${w.newestDateSynced}\x1B[0m  \x1B[90m(last ${formatAge(w.lastSyncAt)})\x1B[0m`);
-			}
-		}
-		console.log();
-	}
-});
-function sortWatermarks(ws) {
-	return [...ws].sort((a, b) => {
-		if (a.table !== b.table) return a.table.localeCompare(b.table);
-		return (a.siteId ?? "").localeCompare(b.siteId ?? "");
-	});
-}
-function sumRows(entries) {
-	return entries.reduce((acc, e) => acc + e.rowCount, 0);
-}
-function sumBytes(entries) {
-	return entries.reduce((acc, e) => acc + e.bytes, 0);
-}
-function formatBytes(n) {
-	if (n < 1024) return `${n} B`;
-	if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
-	if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
-	return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
-}
-const storeCommand = defineCommand({
-	meta: {
-		name: "store",
-		description: "Manage the local DuckDB/Parquet store"
-	},
-	subCommands: {
-		"stats": statsCommand,
-		"compact": compactCommand,
-		"gc": gcCommand,
-		"export": exportCommand,
-		"rollups": rollupsCommand,
-		"rm-site": defineCommand({
-			meta: {
-				name: "rm-site",
-				description: "Delete every parquet, manifest, watermark, and sync-state record for a single site"
-			},
-			args: {
-				site: {
-					type: "positional",
-					required: true,
-					description: "Site URL (e.g. sc-domain:example.com)"
-				},
-				yes: {
-					type: "boolean",
-					alias: "y",
-					default: false,
-					description: "Skip confirmation prompt"
-				},
-				...OUTPUT_ARGS
-			},
-			async run({ args }) {
-				const { json } = applyOutputMode(args);
-				const store = (await createCommandContext({ needsStore: true })).store;
-				const siteId = store.siteIdFor(String(args.site));
-				if (!args.yes && !json) {
-					const ok = await confirm({
-						message: `Delete ALL local data for ${args.site}? This is irreversible.`,
-						initialValue: false
-					});
-					if (isCancel(ok) || !ok) {
-						logger.info("Cancelled");
-						process.exit(0);
-					}
-				}
-				const result = await store.engine.purgeTenant({
-					userId: store.userId,
-					siteId
-				});
-				if (json) {
-					console.log(JSON.stringify(result, null, 2));
-					return;
-				}
-				logger.success(`Removed local data for ${args.site}`);
-				console.log(`  Objects deleted:    ${result.objectsDeleted}`);
-				console.log(`  Manifest entries:   ${result.entriesRemoved}`);
-				console.log(`  Watermarks:         ${result.watermarksRemoved}`);
-				console.log(`  Sync states:        ${result.syncStatesRemoved}`);
-			}
-		}),
-		"reset": defineCommand({
-			meta: {
-				name: "reset",
-				description: "Wipe the entire local store (every site, every table). Irreversible."
-			},
-			args: {
-				yes: {
-					type: "boolean",
-					alias: "y",
-					default: false,
-					description: "Skip confirmation prompt"
-				},
-				...OUTPUT_ARGS
-			},
-			async run({ args }) {
-				const { json } = applyOutputMode(args);
-				const store = (await createCommandContext({ needsStore: true })).store;
-				if (!args.yes && !json) {
-					const ok = await confirm({
-						message: `Wipe the entire local store under ${store.dataDir}? This deletes data for ALL sites.`,
-						initialValue: false
-					});
-					if (isCancel(ok) || !ok) {
-						logger.info("Cancelled");
-						process.exit(0);
-					}
-				}
-				const result = await store.engine.purgeTenant({ userId: store.userId });
-				if (json) {
-					console.log(JSON.stringify(result, null, 2));
-					return;
-				}
-				logger.success("Local store reset");
-				console.log(`  Objects deleted:    ${result.objectsDeleted}`);
-				console.log(`  Manifest entries:   ${result.entriesRemoved}`);
-				console.log(`  Watermarks:         ${result.watermarksRemoved}`);
-				console.log(`  Sync states:        ${result.syncStatesRemoved}`);
-			}
-		})
-	},
-	async run({ args }) {
-		if (!noSubcommandSelected("store", [
-			"stats",
-			"compact",
-			"gc",
-			"export",
-			"rollups",
-			"rm-site",
-			"reset"
-		])) return;
-		await statsCommand.run?.({
-			args,
-			cmd: statsCommand,
-			rawArgs: []
-		});
-	}
-});
-export { storeCommand };