@gscdump/cli 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_chunks/analysis-local.mjs → analysis-local.mjs} +4 -3
- package/dist/{_chunks/auth.mjs → auth.mjs} +2 -2
- package/dist/cli.d.mts +1 -19
- package/dist/cli.mjs +25 -25
- package/dist/{_chunks → commands}/analyze.mjs +3 -3
- package/dist/{_chunks/auth2.mjs → commands/auth.mjs} +4 -4
- package/dist/commands/compact.mjs +128 -0
- package/dist/{_chunks/config2.mjs → commands/config.mjs} +4 -4
- package/dist/{_chunks → commands}/doctor.mjs +8 -7
- package/dist/{_chunks → commands}/dump.mjs +5 -4
- package/dist/{_chunks → commands}/entities.mjs +2 -2
- package/dist/commands/export.mjs +76 -0
- package/dist/commands/gc.mjs +74 -0
- package/dist/{_chunks → commands}/indexing.mjs +3 -3
- package/dist/{_chunks → commands}/init.mjs +5 -5
- package/dist/{_chunks → commands}/inspect.mjs +3 -3
- package/dist/commands/mcp.mjs +57 -0
- package/dist/{_chunks → commands}/profile.mjs +7 -17
- package/dist/{_chunks → commands}/query.mjs +5 -4
- package/dist/{_chunks → commands}/report.mjs +2 -2
- package/dist/commands/rollups.mjs +134 -0
- package/dist/{_chunks → commands}/sitemaps.mjs +3 -3
- package/dist/{_chunks → commands}/sites.mjs +3 -3
- package/dist/commands/stats.mjs +118 -0
- package/dist/commands/store-purge.mjs +93 -0
- package/dist/commands/store.mjs +40 -0
- package/dist/{_chunks → commands}/sync.mjs +3 -2
- package/dist/config.mjs +43 -0
- package/dist/{_chunks/context.mjs → context.mjs} +3 -9
- package/dist/{_chunks/env-file.mjs → env-file.mjs} +3 -3
- package/dist/{_chunks/environment.mjs → environment.mjs} +1 -1
- package/dist/index.d.mts +2 -0
- package/dist/index.mjs +2 -0
- package/dist/local-store.mjs +7 -0
- package/dist/mcp/errors.mjs +85 -0
- package/dist/mcp/handlers/analytics.mjs +1 -0
- package/dist/mcp/handlers/diagnostics.mjs +134 -0
- package/dist/mcp/handlers/index.mjs +7 -0
- package/dist/mcp/handlers/indexing.mjs +27 -0
- package/dist/mcp/handlers/query.mjs +5 -0
- package/dist/mcp/handlers/reports.mjs +72 -0
- package/dist/mcp/handlers/sites.mjs +8 -0
- package/dist/mcp/server/index.mjs +417 -0
- package/dist/mcp/types.mjs +86 -0
- package/dist/package.mjs +2 -0
- package/dist/runtime.d.mts +20 -0
- package/dist/runtime.mjs +38 -0
- package/dist/{_chunks/utils.mjs → utils.mjs} +4 -4
- package/package.json +8 -8
- package/dist/_chunks/config.mjs +0 -93
- package/dist/_chunks/mcp.mjs +0 -867
- package/dist/_chunks/store.mjs +0 -628
- /package/dist/{_chunks/error-handler.mjs → error-handler.mjs} +0 -0
- /package/dist/{_chunks/native-duckdb.mjs → native-duckdb.mjs} +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { loadConfig } from "
|
|
2
|
-
import { ALL_SEARCH_TYPES, exportToCSV, logger, parseSearchType } from "
|
|
3
|
-
import { allTables,
|
|
4
|
-
import {
|
|
1
|
+
import { loadConfig } from "../config.mjs";
|
|
2
|
+
import { ALL_SEARCH_TYPES, exportToCSV, logger, parseSearchType } from "../utils.mjs";
|
|
3
|
+
import { allTables, inferTable } from "../local-store.mjs";
|
|
4
|
+
import { createCommandContext } from "../context.mjs";
|
|
5
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
5
6
|
import process from "node:process";
|
|
6
7
|
import { defineCommand } from "citty";
|
|
7
8
|
import fs from "node:fs/promises";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { logger } from "
|
|
2
|
-
import { resolveAnalysisSource } from "
|
|
1
|
+
import { logger } from "../utils.mjs";
|
|
2
|
+
import { resolveAnalysisSource } from "../analysis-local.mjs";
|
|
3
3
|
import { defineCommand } from "citty";
|
|
4
4
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
5
5
|
import { err, ok, unwrapResult } from "gscdump/result";
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
import { INTENT_CLASSIFIER_VERSION, NORMALIZER_VERSION, classifyQueryIntent, encodeIntent, normalizeQuery } from "@gscdump/analysis";
|
|
5
|
+
import { buildQueryDimRecords, createQueryDimStore } from "@gscdump/engine/entities";
|
|
6
|
+
import { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/engine/rollups";
|
|
7
|
+
async function buildSiteQueryDim(store, siteId) {
|
|
8
|
+
const ctx = {
|
|
9
|
+
userId: store.userId,
|
|
10
|
+
siteId
|
|
11
|
+
};
|
|
12
|
+
const entries = await store.engine.listLive({
|
|
13
|
+
userId: ctx.userId,
|
|
14
|
+
siteId,
|
|
15
|
+
table: "queries"
|
|
16
|
+
});
|
|
17
|
+
if (entries.length === 0) return 0;
|
|
18
|
+
const { rows } = await store.engine.runSQL({
|
|
19
|
+
ctx,
|
|
20
|
+
table: "queries",
|
|
21
|
+
fileSets: { FILES: {
|
|
22
|
+
table: "queries",
|
|
23
|
+
partitions: entries.map((e) => e.partition)
|
|
24
|
+
} },
|
|
25
|
+
sql: `SELECT DISTINCT query FROM read_parquet({{FILES}}, union_by_name = true) WHERE query IS NOT NULL`
|
|
26
|
+
});
|
|
27
|
+
const records = buildQueryDimRecords(rows.map((r) => String(r.query)), {
|
|
28
|
+
normalizeQuery,
|
|
29
|
+
normalizerVersion: NORMALIZER_VERSION,
|
|
30
|
+
classifyIntentCode: (q) => encodeIntent(classifyQueryIntent(q)),
|
|
31
|
+
intentVersion: INTENT_CLASSIFIER_VERSION
|
|
32
|
+
});
|
|
33
|
+
await createQueryDimStore({ dataSource: store.dataSource }).write(ctx, records, Date.now());
|
|
34
|
+
return records.length;
|
|
35
|
+
}
|
|
36
|
+
const rollupsCommand = defineCommand({
|
|
37
|
+
meta: {
|
|
38
|
+
name: "rollups",
|
|
39
|
+
description: "Manage post-sync rollups"
|
|
40
|
+
},
|
|
41
|
+
subCommands: { rebuild: defineCommand({
|
|
42
|
+
meta: {
|
|
43
|
+
name: "rebuild",
|
|
44
|
+
description: "Rebuild post-sync rollups (daily totals, weekly totals, top-N tables) for a site"
|
|
45
|
+
},
|
|
46
|
+
args: {
|
|
47
|
+
...OUTPUT_ARGS,
|
|
48
|
+
"site": {
|
|
49
|
+
type: "string",
|
|
50
|
+
alias: "s",
|
|
51
|
+
description: "Restrict to a single site (default: all sites with local data)"
|
|
52
|
+
},
|
|
53
|
+
"with-canonical": {
|
|
54
|
+
type: "boolean",
|
|
55
|
+
description: "Also build the opt-in canonical-primary rollups (query_canonical_variants, query_canonical_daily)",
|
|
56
|
+
default: false
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
async run({ args }) {
|
|
60
|
+
const { json } = applyOutputMode(args);
|
|
61
|
+
const defs = args["with-canonical"] ? [...DEFAULT_ROLLUPS, ...CANONICAL_ROLLUPS] : DEFAULT_ROLLUPS;
|
|
62
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
63
|
+
const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
64
|
+
const allSiteIds = /* @__PURE__ */ new Set();
|
|
65
|
+
if (explicitSiteId) allSiteIds.add(explicitSiteId);
|
|
66
|
+
else {
|
|
67
|
+
const entries = await store.engine.listLive({ userId: store.userId });
|
|
68
|
+
for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
|
|
69
|
+
}
|
|
70
|
+
if (allSiteIds.size === 0) {
|
|
71
|
+
if (json) console.log(JSON.stringify({
|
|
72
|
+
sites: [],
|
|
73
|
+
totalBytes: 0
|
|
74
|
+
}, null, 2));
|
|
75
|
+
else logger.warn("No sites with local data. Run `gscdump sync` first.");
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const summary = [];
|
|
79
|
+
let totalBytes = 0;
|
|
80
|
+
for (const siteId of allSiteIds) {
|
|
81
|
+
if (args["with-canonical"]) {
|
|
82
|
+
const dimRows = await buildSiteQueryDim(store, siteId);
|
|
83
|
+
if (!json) logger.info(`Built query dimension for [${siteId}] (${dimRows} distinct queries, normalizer v${NORMALIZER_VERSION})`);
|
|
84
|
+
}
|
|
85
|
+
logger.info(`Rebuilding rollups for [${siteId}] (${defs.length} rollups)`);
|
|
86
|
+
const results = await rebuildRollups({
|
|
87
|
+
engine: {
|
|
88
|
+
runSQL: (opts) => store.engine.runSQL(opts),
|
|
89
|
+
listPartitions: async ({ ctx, table, searchType }) => {
|
|
90
|
+
return (await store.engine.listLive({
|
|
91
|
+
userId: ctx.userId,
|
|
92
|
+
...ctx.siteId !== void 0 ? { siteId: ctx.siteId } : {},
|
|
93
|
+
table,
|
|
94
|
+
...searchType !== void 0 ? { searchType } : {}
|
|
95
|
+
})).map((e) => ({
|
|
96
|
+
partition: e.partition,
|
|
97
|
+
bytes: e.bytes
|
|
98
|
+
}));
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
dataSource: store.dataSource,
|
|
102
|
+
ctx: {
|
|
103
|
+
userId: store.userId,
|
|
104
|
+
siteId
|
|
105
|
+
},
|
|
106
|
+
defs
|
|
107
|
+
});
|
|
108
|
+
const site = {
|
|
109
|
+
siteId,
|
|
110
|
+
rollups: []
|
|
111
|
+
};
|
|
112
|
+
for (const r of results) {
|
|
113
|
+
totalBytes += r.bytes;
|
|
114
|
+
site.rollups.push({
|
|
115
|
+
id: r.id,
|
|
116
|
+
bytes: r.bytes,
|
|
117
|
+
objectKey: r.objectKey
|
|
118
|
+
});
|
|
119
|
+
if (!json) console.log(` ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB ${r.objectKey}`);
|
|
120
|
+
}
|
|
121
|
+
summary.push(site);
|
|
122
|
+
}
|
|
123
|
+
if (json) {
|
|
124
|
+
console.log(JSON.stringify({
|
|
125
|
+
sites: summary,
|
|
126
|
+
totalBytes
|
|
127
|
+
}, null, 2));
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
|
|
131
|
+
}
|
|
132
|
+
}) }
|
|
133
|
+
});
|
|
134
|
+
export { rollupsCommand };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected } from "
|
|
2
|
-
import { createCommandContext } from "
|
|
3
|
-
import { gscErrorHandler } from "
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
4
4
|
import process from "node:process";
|
|
5
5
|
import { defineCommand } from "citty";
|
|
6
6
|
import { discoverSitemap, fetchSitemap, fetchSitemapUrls } from "gscdump";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, logger } from "
|
|
2
|
-
import { createCommandContext } from "
|
|
3
|
-
import { gscErrorHandler } from "
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
4
4
|
import process from "node:process";
|
|
5
5
|
import { defineCommand } from "citty";
|
|
6
6
|
import { confirm, isCancel } from "@clack/prompts";
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath, formatAge, logger } from "../utils.mjs";
|
|
2
|
+
import { allTables } from "../local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "../context.mjs";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { defineCommand } from "citty";
|
|
6
|
+
import { filesystemStats } from "@gscdump/engine/filesystem";
|
|
7
|
+
const statsCommand = defineCommand({
|
|
8
|
+
meta: {
|
|
9
|
+
name: "stats",
|
|
10
|
+
description: "Show row/byte counts per table and on-disk footprint"
|
|
11
|
+
},
|
|
12
|
+
args: {
|
|
13
|
+
...OUTPUT_ARGS,
|
|
14
|
+
site: {
|
|
15
|
+
type: "string",
|
|
16
|
+
description: "Limit to one site URL (sc-domain:example.com, https://example.com/, ...)"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
async run({ args }) {
|
|
20
|
+
const { json } = applyOutputMode(args);
|
|
21
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
22
|
+
const allEntries = await store.engine.listAll({ userId: store.userId });
|
|
23
|
+
let siteId;
|
|
24
|
+
if (args.site) {
|
|
25
|
+
const known = new Set(allEntries.filter((entry) => entry.retiredAt === void 0 && entry.siteId !== void 0).map((entry) => entry.siteId));
|
|
26
|
+
const candidate = store.siteIdFor(args.site);
|
|
27
|
+
if (!known.has(candidate)) {
|
|
28
|
+
logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
siteId = candidate;
|
|
32
|
+
}
|
|
33
|
+
const selectedEntries = siteId === void 0 ? allEntries : allEntries.filter((entry) => entry.siteId === siteId);
|
|
34
|
+
const perTable = allTables().map((table) => {
|
|
35
|
+
const all = selectedEntries.filter((entry) => entry.table === table);
|
|
36
|
+
return {
|
|
37
|
+
table,
|
|
38
|
+
live: all.filter((e) => e.retiredAt === void 0),
|
|
39
|
+
retired: all.filter((e) => e.retiredAt !== void 0)
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
const [watermarks, disk] = await Promise.all([store.engine.getWatermarks({
|
|
43
|
+
userId: store.userId,
|
|
44
|
+
siteId
|
|
45
|
+
}), filesystemStats(store.dataDir).catch(() => ({
|
|
46
|
+
files: 0,
|
|
47
|
+
bytes: 0
|
|
48
|
+
}))]);
|
|
49
|
+
if (json) {
|
|
50
|
+
const payload = {
|
|
51
|
+
dataDir: store.dataDir,
|
|
52
|
+
disk,
|
|
53
|
+
tables: perTable.map(({ table, live, retired }) => ({
|
|
54
|
+
table,
|
|
55
|
+
liveFiles: live.length,
|
|
56
|
+
liveRows: sumRows(live),
|
|
57
|
+
liveBytes: sumBytes(live),
|
|
58
|
+
retiredFiles: retired.length,
|
|
59
|
+
retiredBytes: sumBytes(retired),
|
|
60
|
+
watermarks: watermarks.filter((w) => w.table === table).map((w) => ({
|
|
61
|
+
siteId: w.siteId ?? null,
|
|
62
|
+
newestDateSynced: w.newestDateSynced,
|
|
63
|
+
oldestDateSynced: w.oldestDateSynced,
|
|
64
|
+
lastSyncAt: w.lastSyncAt
|
|
65
|
+
}))
|
|
66
|
+
}))
|
|
67
|
+
};
|
|
68
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
console.log();
|
|
72
|
+
console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
|
|
73
|
+
console.log(` \x1B[90mDisk: ${disk.files} file(s), ${formatBytes(disk.bytes)}\x1B[0m`);
|
|
74
|
+
console.log();
|
|
75
|
+
const totalRows = perTable.reduce((acc, t) => acc + sumRows(t.live), 0);
|
|
76
|
+
const totalBytes = perTable.reduce((acc, t) => acc + sumBytes(t.live), 0);
|
|
77
|
+
const totalFiles = perTable.reduce((acc, t) => acc + t.live.length, 0);
|
|
78
|
+
const totalRetiredFiles = perTable.reduce((acc, t) => acc + t.retired.length, 0);
|
|
79
|
+
const totalRetiredBytes = perTable.reduce((acc, t) => acc + sumBytes(t.retired), 0);
|
|
80
|
+
for (const { table, live, retired } of perTable) {
|
|
81
|
+
const rows = sumRows(live).toLocaleString();
|
|
82
|
+
const bytes = formatBytes(sumBytes(live));
|
|
83
|
+
const retiredSuffix = retired.length > 0 ? ` \x1B[90m(+${retired.length} retired, ${formatBytes(sumBytes(retired))})\x1B[0m` : "";
|
|
84
|
+
console.log(` ${table.padEnd(15)} \x1B[36m${String(live.length).padStart(4)}\x1B[0m files, ${rows.padStart(10)} rows, ${bytes}${retiredSuffix}`);
|
|
85
|
+
}
|
|
86
|
+
console.log();
|
|
87
|
+
console.log(` \x1B[1mTotal:\x1B[0m ${totalFiles} files, ${totalRows.toLocaleString()} rows, ${formatBytes(totalBytes)} live`);
|
|
88
|
+
if (totalRetiredFiles > 0) console.log(` \x1B[90mRetired: ${totalRetiredFiles} files, ${formatBytes(totalRetiredBytes)} awaiting GC\x1B[0m`);
|
|
89
|
+
if (watermarks.length > 0) {
|
|
90
|
+
console.log();
|
|
91
|
+
console.log(` \x1B[1mSync watermarks:\x1B[0m`);
|
|
92
|
+
for (const w of sortWatermarks(watermarks)) {
|
|
93
|
+
const scope = w.siteId ? `${w.table}@${w.siteId}` : w.table;
|
|
94
|
+
console.log(` ${scope.padEnd(24)} \x1B[36m${w.oldestDateSynced}\x1B[0m → \x1B[36m${w.newestDateSynced}\x1B[0m \x1B[90m(last ${formatAge(w.lastSyncAt)})\x1B[0m`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
console.log();
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
function sortWatermarks(ws) {
|
|
101
|
+
return [...ws].sort((a, b) => {
|
|
102
|
+
if (a.table !== b.table) return a.table.localeCompare(b.table);
|
|
103
|
+
return (a.siteId ?? "").localeCompare(b.siteId ?? "");
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
function sumRows(entries) {
|
|
107
|
+
return entries.reduce((acc, e) => acc + e.rowCount, 0);
|
|
108
|
+
}
|
|
109
|
+
function sumBytes(entries) {
|
|
110
|
+
return entries.reduce((acc, e) => acc + e.bytes, 0);
|
|
111
|
+
}
|
|
112
|
+
function formatBytes(n) {
|
|
113
|
+
if (n < 1024) return `${n} B`;
|
|
114
|
+
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
|
115
|
+
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
|
|
116
|
+
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
|
|
117
|
+
}
|
|
118
|
+
export { statsCommand };
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import process from "node:process";
|
|
4
|
+
import { defineCommand } from "citty";
|
|
5
|
+
import { confirm, isCancel } from "@clack/prompts";
|
|
6
|
+
const rmSiteCommand = defineCommand({
|
|
7
|
+
meta: {
|
|
8
|
+
name: "rm-site",
|
|
9
|
+
description: "Delete every parquet, manifest, watermark, and sync-state record for a single site"
|
|
10
|
+
},
|
|
11
|
+
args: {
|
|
12
|
+
site: {
|
|
13
|
+
type: "positional",
|
|
14
|
+
required: true,
|
|
15
|
+
description: "Site URL (e.g. sc-domain:example.com)"
|
|
16
|
+
},
|
|
17
|
+
yes: {
|
|
18
|
+
type: "boolean",
|
|
19
|
+
alias: "y",
|
|
20
|
+
default: false,
|
|
21
|
+
description: "Skip confirmation prompt"
|
|
22
|
+
},
|
|
23
|
+
...OUTPUT_ARGS
|
|
24
|
+
},
|
|
25
|
+
async run({ args }) {
|
|
26
|
+
const { json } = applyOutputMode(args);
|
|
27
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
28
|
+
const siteId = store.siteIdFor(String(args.site));
|
|
29
|
+
if (!args.yes && !json) {
|
|
30
|
+
const ok = await confirm({
|
|
31
|
+
message: `Delete ALL local data for ${args.site}? This is irreversible.`,
|
|
32
|
+
initialValue: false
|
|
33
|
+
});
|
|
34
|
+
if (isCancel(ok) || !ok) {
|
|
35
|
+
logger.info("Cancelled");
|
|
36
|
+
process.exit(0);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const result = await store.engine.purgeTenant({
|
|
40
|
+
userId: store.userId,
|
|
41
|
+
siteId
|
|
42
|
+
});
|
|
43
|
+
if (json) {
|
|
44
|
+
console.log(JSON.stringify(result, null, 2));
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
logger.success(`Removed local data for ${args.site}`);
|
|
48
|
+
console.log(` Objects deleted: ${result.objectsDeleted}`);
|
|
49
|
+
console.log(` Manifest entries: ${result.entriesRemoved}`);
|
|
50
|
+
console.log(` Watermarks: ${result.watermarksRemoved}`);
|
|
51
|
+
console.log(` Sync states: ${result.syncStatesRemoved}`);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
const resetCommand = defineCommand({
|
|
55
|
+
meta: {
|
|
56
|
+
name: "reset",
|
|
57
|
+
description: "Wipe the entire local store (every site, every table). Irreversible."
|
|
58
|
+
},
|
|
59
|
+
args: {
|
|
60
|
+
yes: {
|
|
61
|
+
type: "boolean",
|
|
62
|
+
alias: "y",
|
|
63
|
+
default: false,
|
|
64
|
+
description: "Skip confirmation prompt"
|
|
65
|
+
},
|
|
66
|
+
...OUTPUT_ARGS
|
|
67
|
+
},
|
|
68
|
+
async run({ args }) {
|
|
69
|
+
const { json } = applyOutputMode(args);
|
|
70
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
71
|
+
if (!args.yes && !json) {
|
|
72
|
+
const ok = await confirm({
|
|
73
|
+
message: `Wipe the entire local store under ${store.dataDir}? This deletes data for ALL sites.`,
|
|
74
|
+
initialValue: false
|
|
75
|
+
});
|
|
76
|
+
if (isCancel(ok) || !ok) {
|
|
77
|
+
logger.info("Cancelled");
|
|
78
|
+
process.exit(0);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const result = await store.engine.purgeTenant({ userId: store.userId });
|
|
82
|
+
if (json) {
|
|
83
|
+
console.log(JSON.stringify(result, null, 2));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
logger.success("Local store reset");
|
|
87
|
+
console.log(` Objects deleted: ${result.objectsDeleted}`);
|
|
88
|
+
console.log(` Manifest entries: ${result.entriesRemoved}`);
|
|
89
|
+
console.log(` Watermarks: ${result.watermarksRemoved}`);
|
|
90
|
+
console.log(` Sync states: ${result.syncStatesRemoved}`);
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
export { resetCommand, rmSiteCommand };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { noSubcommandSelected } from "../utils.mjs";
|
|
2
|
+
import { compactCommand } from "./compact.mjs";
|
|
3
|
+
import { exportCommand } from "./export.mjs";
|
|
4
|
+
import { gcCommand } from "./gc.mjs";
|
|
5
|
+
import { rollupsCommand } from "./rollups.mjs";
|
|
6
|
+
import { statsCommand } from "./stats.mjs";
|
|
7
|
+
import { resetCommand, rmSiteCommand } from "./store-purge.mjs";
|
|
8
|
+
import { defineCommand } from "citty";
|
|
9
|
+
const storeCommand = defineCommand({
|
|
10
|
+
meta: {
|
|
11
|
+
name: "store",
|
|
12
|
+
description: "Manage the local DuckDB/Parquet store"
|
|
13
|
+
},
|
|
14
|
+
subCommands: {
|
|
15
|
+
"stats": statsCommand,
|
|
16
|
+
"compact": compactCommand,
|
|
17
|
+
"gc": gcCommand,
|
|
18
|
+
"export": exportCommand,
|
|
19
|
+
"rollups": rollupsCommand,
|
|
20
|
+
"rm-site": rmSiteCommand,
|
|
21
|
+
"reset": resetCommand
|
|
22
|
+
},
|
|
23
|
+
async run({ args }) {
|
|
24
|
+
if (!noSubcommandSelected("store", [
|
|
25
|
+
"stats",
|
|
26
|
+
"compact",
|
|
27
|
+
"gc",
|
|
28
|
+
"export",
|
|
29
|
+
"rollups",
|
|
30
|
+
"rm-site",
|
|
31
|
+
"reset"
|
|
32
|
+
])) return;
|
|
33
|
+
await statsCommand.run?.({
|
|
34
|
+
args,
|
|
35
|
+
cmd: statsCommand,
|
|
36
|
+
rawArgs: []
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
export { storeCommand };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, clearLine, displayPath, formatAge, logger, progressBar, runWithConcurrency } from "
|
|
2
|
-
import { TABLE_DIMS, allTables,
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, clearLine, displayPath, formatAge, logger, progressBar, runWithConcurrency } from "../utils.mjs";
|
|
2
|
+
import { TABLE_DIMS, allTables, createLocalStore, transformGscRow } from "../local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
4
|
import process from "node:process";
|
|
4
5
|
import { defineCommand } from "citty";
|
|
5
6
|
import { SearchTypes } from "gscdump/query";
|
package/dist/config.mjs
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { useCliRuntime } from "./runtime.mjs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import fs from "node:fs/promises";
|
|
5
|
+
function setConfigDir(dir) {
|
|
6
|
+
useCliRuntime().configDir = dir;
|
|
7
|
+
}
|
|
8
|
+
function getConfigDir() {
|
|
9
|
+
return useCliRuntime().configDir;
|
|
10
|
+
}
|
|
11
|
+
function defaultDataDir() {
|
|
12
|
+
return path.join(os.homedir(), ".gscdump", "data");
|
|
13
|
+
}
|
|
14
|
+
function resolveDataDir(config) {
|
|
15
|
+
return expandTilde(config.dataDir ?? defaultDataDir());
|
|
16
|
+
}
|
|
17
|
+
function expandTilde(p) {
|
|
18
|
+
if (p === "~") return os.homedir();
|
|
19
|
+
if (p.startsWith("~/")) return path.join(os.homedir(), p.slice(2));
|
|
20
|
+
return p;
|
|
21
|
+
}
|
|
22
|
+
async function loadConfig() {
|
|
23
|
+
return fs.readFile(path.join(getConfigDir(), "config.json"), "utf-8").then((data) => JSON.parse(data)).catch(() => ({}));
|
|
24
|
+
}
|
|
25
|
+
async function loadResolvedConfig() {
|
|
26
|
+
const config = await loadConfig();
|
|
27
|
+
return {
|
|
28
|
+
config,
|
|
29
|
+
dataDir: resolveDataDir(config)
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
async function saveConfig(config) {
|
|
33
|
+
const configDir = getConfigDir();
|
|
34
|
+
await fs.mkdir(configDir, {
|
|
35
|
+
recursive: true,
|
|
36
|
+
mode: 448
|
|
37
|
+
});
|
|
38
|
+
await fs.writeFile(path.join(configDir, "config.json"), JSON.stringify(config, null, 2), { mode: 384 });
|
|
39
|
+
}
|
|
40
|
+
function getConfigPath() {
|
|
41
|
+
return path.join(getConfigDir(), "config.json");
|
|
42
|
+
}
|
|
43
|
+
export { defaultDataDir, getConfigDir, getConfigPath, loadConfig, loadResolvedConfig, resolveDataDir, saveConfig, setConfigDir };
|
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { loadResolvedConfig } from "./config.mjs";
|
|
2
2
|
import { logger } from "./utils.mjs";
|
|
3
3
|
import { resolveAuth } from "./auth.mjs";
|
|
4
|
+
import { createLocalStore } from "./local-store.mjs";
|
|
4
5
|
import process from "node:process";
|
|
5
6
|
import { cancel, isCancel, select } from "@clack/prompts";
|
|
6
7
|
import { googleSearchConsole } from "gscdump";
|
|
7
|
-
import { createNodeHarness } from "@gscdump/engine/node";
|
|
8
|
-
import { TABLE_DIMS, transformGscRow } from "@gscdump/engine/ingest";
|
|
9
|
-
import { allTables, inferTable } from "@gscdump/engine/schema";
|
|
10
|
-
function createLocalStore(opts) {
|
|
11
|
-
return createNodeHarness(opts);
|
|
12
|
-
}
|
|
13
|
-
var context_exports = /* @__PURE__ */ __exportAll({ createCommandContext: () => createCommandContext });
|
|
14
8
|
async function createCommandContext(opts = {}) {
|
|
15
9
|
const { needsAuth = false, needsStore = false, interactive = false, byok, fetchOptions } = opts;
|
|
16
10
|
const { config, dataDir } = await loadResolvedConfig();
|
|
@@ -66,4 +60,4 @@ async function createCommandContext(opts = {}) {
|
|
|
66
60
|
resolveSite
|
|
67
61
|
};
|
|
68
62
|
}
|
|
69
|
-
export {
|
|
63
|
+
export { createCommandContext };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { useCliRuntime } from "./
|
|
1
|
+
import { useCliRuntime } from "./runtime.mjs";
|
|
2
|
+
import path from "node:path";
|
|
2
3
|
import process from "node:process";
|
|
3
4
|
import fs from "node:fs";
|
|
4
|
-
import path from "node:path";
|
|
5
5
|
const ENV_LINE_RE = /^([^=]+)=(.*)$/;
|
|
6
6
|
function parseEnvFile(envPath) {
|
|
7
7
|
let content;
|
|
@@ -50,4 +50,4 @@ function loadEnvFile(opts) {
|
|
|
50
50
|
}
|
|
51
51
|
return applied;
|
|
52
52
|
}
|
|
53
|
-
export { getAppliedEnvKeys, getLoadedEnvPath, loadEnvFromCwd, parseEnvFile };
|
|
53
|
+
export { getAppliedEnvKeys, getLoadedEnvPath, loadEnvFile, loadEnvFromCwd, parseEnvFile };
|
package/dist/index.d.mts
ADDED
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { createNodeHarness } from "@gscdump/engine/node";
|
|
2
|
+
import { TABLE_DIMS, transformGscRow } from "@gscdump/engine/ingest";
|
|
3
|
+
import { allTables, inferTable } from "@gscdump/engine/schema";
|
|
4
|
+
function createLocalStore(opts) {
|
|
5
|
+
return createNodeHarness(opts);
|
|
6
|
+
}
|
|
7
|
+
export { TABLE_DIMS, allTables, createLocalStore, inferTable, transformGscRow };
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { isQueryError } from "gscdump/query";
|
|
2
|
+
import { isAnalysisError } from "@gscdump/analysis/errors";
|
|
3
|
+
import { isEngineError } from "@gscdump/engine/errors";
|
|
4
|
+
const PERIODS = [
|
|
5
|
+
"7d",
|
|
6
|
+
"28d",
|
|
7
|
+
"30d",
|
|
8
|
+
"90d",
|
|
9
|
+
"180d",
|
|
10
|
+
"365d",
|
|
11
|
+
"mtd",
|
|
12
|
+
"ytd",
|
|
13
|
+
"custom"
|
|
14
|
+
];
|
|
15
|
+
const COMPARISONS = [
|
|
16
|
+
"none",
|
|
17
|
+
"prev-period",
|
|
18
|
+
"yoy"
|
|
19
|
+
];
|
|
20
|
+
const mcpHandlerErrors = {
|
|
21
|
+
unknownReport(id, available) {
|
|
22
|
+
return {
|
|
23
|
+
kind: "unknown-report",
|
|
24
|
+
id,
|
|
25
|
+
available,
|
|
26
|
+
message: `Unknown report id "${id}". Available: ${available.join(", ")}`
|
|
27
|
+
};
|
|
28
|
+
},
|
|
29
|
+
unknownPeriod(value) {
|
|
30
|
+
return {
|
|
31
|
+
kind: "unknown-period",
|
|
32
|
+
value,
|
|
33
|
+
supported: PERIODS,
|
|
34
|
+
message: `Unknown period "${value}". Supported: ${PERIODS.join(", ")}.`
|
|
35
|
+
};
|
|
36
|
+
},
|
|
37
|
+
unknownComparison(value) {
|
|
38
|
+
return {
|
|
39
|
+
kind: "unknown-comparison",
|
|
40
|
+
value,
|
|
41
|
+
supported: COMPARISONS,
|
|
42
|
+
message: `Unknown comparison "${value}". Supported: ${COMPARISONS.join(", ")}.`
|
|
43
|
+
};
|
|
44
|
+
},
|
|
45
|
+
noValidDimension() {
|
|
46
|
+
return {
|
|
47
|
+
kind: "no-valid-dimension",
|
|
48
|
+
message: "At least one valid dimension required"
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
function mcpHandlerErrorToException(error) {
|
|
53
|
+
const exception = new Error(error.message);
|
|
54
|
+
exception.mcpHandlerError = error;
|
|
55
|
+
return exception;
|
|
56
|
+
}
|
|
57
|
+
function enrichToolError(error) {
|
|
58
|
+
const query = asQueryError(error);
|
|
59
|
+
if (query) return /* @__PURE__ */ new Error(`[query:${query.kind}] ${query.message}`);
|
|
60
|
+
const analysis = asAnalysisError(error);
|
|
61
|
+
if (analysis) {
|
|
62
|
+
const cause = analysis.kind === "required-step-failed" ? asEngineError(analysis.cause) : null;
|
|
63
|
+
const tail = cause ? ` (engine:${cause.kind})` : "";
|
|
64
|
+
return /* @__PURE__ */ new Error(`[analysis:${analysis.kind}]${tail} ${analysis.message}`);
|
|
65
|
+
}
|
|
66
|
+
const engine = asEngineError(error);
|
|
67
|
+
if (engine) return /* @__PURE__ */ new Error(`[engine:${engine.kind}] ${engine.message}`);
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
function asQueryError(error) {
|
|
71
|
+
if (isQueryError(error)) return error;
|
|
72
|
+
const tagged = error?.queryError;
|
|
73
|
+
return isQueryError(tagged) ? tagged : null;
|
|
74
|
+
}
|
|
75
|
+
function asEngineError(error) {
|
|
76
|
+
if (isEngineError(error)) return error;
|
|
77
|
+
const tagged = error?.engineError;
|
|
78
|
+
return isEngineError(tagged) ? tagged : null;
|
|
79
|
+
}
|
|
80
|
+
function asAnalysisError(error) {
|
|
81
|
+
if (isAnalysisError(error)) return error;
|
|
82
|
+
const tagged = error?.analysisError;
|
|
83
|
+
return isAnalysisError(tagged) ? tagged : null;
|
|
84
|
+
}
|
|
85
|
+
export { enrichToolError, mcpHandlerErrorToException, mcpHandlerErrors };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|