@gscdump/cli 1.3.2 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_chunks/analysis-local.mjs → analysis-local.mjs} +4 -3
- package/dist/{_chunks/auth.mjs → auth.mjs} +2 -2
- package/dist/cli.d.mts +1 -19
- package/dist/cli.mjs +25 -25
- package/dist/{_chunks → commands}/analyze.mjs +3 -3
- package/dist/{_chunks/auth2.mjs → commands/auth.mjs} +4 -4
- package/dist/commands/compact.mjs +128 -0
- package/dist/{_chunks/config2.mjs → commands/config.mjs} +4 -4
- package/dist/{_chunks → commands}/doctor.mjs +8 -7
- package/dist/{_chunks → commands}/dump.mjs +5 -4
- package/dist/{_chunks → commands}/entities.mjs +2 -2
- package/dist/commands/export.mjs +76 -0
- package/dist/commands/gc.mjs +74 -0
- package/dist/{_chunks → commands}/indexing.mjs +3 -3
- package/dist/{_chunks → commands}/init.mjs +5 -5
- package/dist/{_chunks → commands}/inspect.mjs +3 -3
- package/dist/commands/mcp.mjs +57 -0
- package/dist/{_chunks → commands}/profile.mjs +7 -17
- package/dist/{_chunks → commands}/query.mjs +5 -4
- package/dist/{_chunks → commands}/report.mjs +2 -2
- package/dist/commands/rollups.mjs +134 -0
- package/dist/{_chunks → commands}/sitemaps.mjs +3 -3
- package/dist/{_chunks → commands}/sites.mjs +3 -3
- package/dist/commands/stats.mjs +118 -0
- package/dist/commands/store-purge.mjs +93 -0
- package/dist/commands/store.mjs +40 -0
- package/dist/{_chunks → commands}/sync.mjs +3 -2
- package/dist/config.mjs +43 -0
- package/dist/{_chunks/context.mjs → context.mjs} +3 -9
- package/dist/{_chunks/env-file.mjs → env-file.mjs} +3 -3
- package/dist/{_chunks/environment.mjs → environment.mjs} +1 -1
- package/dist/index.d.mts +2 -0
- package/dist/index.mjs +2 -0
- package/dist/local-store.mjs +7 -0
- package/dist/mcp/errors.mjs +85 -0
- package/dist/mcp/handlers/analytics.mjs +1 -0
- package/dist/mcp/handlers/diagnostics.mjs +134 -0
- package/dist/mcp/handlers/index.mjs +7 -0
- package/dist/mcp/handlers/indexing.mjs +27 -0
- package/dist/mcp/handlers/query.mjs +5 -0
- package/dist/mcp/handlers/reports.mjs +72 -0
- package/dist/mcp/handlers/sites.mjs +8 -0
- package/dist/mcp/server/index.mjs +417 -0
- package/dist/mcp/types.mjs +86 -0
- package/dist/package.mjs +2 -0
- package/dist/runtime.d.mts +20 -0
- package/dist/runtime.mjs +38 -0
- package/dist/{_chunks/utils.mjs → utils.mjs} +4 -4
- package/package.json +8 -8
- package/dist/_chunks/config.mjs +0 -93
- package/dist/_chunks/mcp.mjs +0 -867
- package/dist/_chunks/store.mjs +0 -628
- /package/dist/{_chunks/error-handler.mjs → error-handler.mjs} +0 -0
- /package/dist/{_chunks/native-duckdb.mjs → native-duckdb.mjs} +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { logger } from "./utils.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { createLocalStore } from "./local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "./context.mjs";
|
|
3
4
|
import { LocalStoreUnsupportedError } from "./error-handler.mjs";
|
|
5
|
+
import { join } from "node:path";
|
|
4
6
|
import process from "node:process";
|
|
5
7
|
import { readdir } from "node:fs/promises";
|
|
6
|
-
import { join } from "node:path";
|
|
7
8
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
8
9
|
import { AnalyzerCapabilityError, createEngineQuerySource, runAnalyzerFromSource } from "@gscdump/analysis";
|
|
9
10
|
import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
|
|
@@ -85,4 +86,4 @@ async function resolveAnalysisSource(args) {
|
|
|
85
86
|
runAnalysis: makeRunAnalysis(source, "live")
|
|
86
87
|
};
|
|
87
88
|
}
|
|
88
|
-
export { resolveAnalysisSource };
|
|
89
|
+
export { hasLocalData, listLocalSites, resolveAnalysisSource };
|
|
@@ -2,9 +2,9 @@ import { getConfigDir, loadConfig } from "./config.mjs";
|
|
|
2
2
|
import { pickCliEnvironmentValue, resolveCliEnvironment } from "./environment.mjs";
|
|
3
3
|
import { displayPath, logger } from "./utils.mjs";
|
|
4
4
|
import { getAppliedEnvKeys, getLoadedEnvPath } from "./env-file.mjs";
|
|
5
|
+
import path from "node:path";
|
|
5
6
|
import process from "node:process";
|
|
6
7
|
import fs from "node:fs/promises";
|
|
7
|
-
import path from "node:path";
|
|
8
8
|
import { isCancel, text } from "@clack/prompts";
|
|
9
9
|
import { err, ok, unwrapResult } from "gscdump/result";
|
|
10
10
|
import { createAuth } from "gscdump";
|
|
@@ -473,4 +473,4 @@ function isAuthError(err) {
|
|
|
473
473
|
if (!msg) return false;
|
|
474
474
|
return /\b(?:401|403|unauthorized|forbidden|invalid_grant|invalid_token|insufficient.*scope|invalid_client|token has been expired|token has been revoked)\b/.test(msg) || msg.includes("oauth2.googleapis.com/token");
|
|
475
475
|
}
|
|
476
|
-
export { authenticate, clearTokens, formatAuthProvenance, getAuth, getAuthCredentials, isAuthError, loadServiceAccount, loadTokens, resolveAuth, resolveBYOK, resolveServiceAccount, saveTokens };
|
|
476
|
+
export { authenticate, authenticateDeviceCode, authenticateDeviceCodeResult, clearTokens, describeAuthProvenance, formatAuthProvenance, getAuth, getAuthCredentials, isAuthError, loadServiceAccount, loadServiceAccountResult, loadTokens, resolveAuth, resolveBYOK, resolveServiceAccount, saveTokens };
|
package/dist/cli.d.mts
CHANGED
|
@@ -1,22 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type CliEnvironmentSource = Record<string, string | undefined>;
|
|
3
|
-
interface CliRuntime {
|
|
4
|
-
activeProfileOverride: string | null;
|
|
5
|
-
colorEnabled: boolean;
|
|
6
|
-
configDir: string;
|
|
7
|
-
configDirOverridden: boolean;
|
|
8
|
-
environment: CliEnvironmentSource;
|
|
9
|
-
logger: ConsolaInstance;
|
|
10
|
-
quiet: boolean;
|
|
11
|
-
rawArgs: string[];
|
|
12
|
-
}
|
|
13
|
-
interface CreateCliRuntimeOptions {
|
|
14
|
-
configDir?: string;
|
|
15
|
-
environment?: CliEnvironmentSource;
|
|
16
|
-
rawArgs?: readonly string[];
|
|
17
|
-
stderr?: NodeJS.WriteStream;
|
|
18
|
-
}
|
|
19
|
-
declare function createCliRuntime(opts?: CreateCliRuntimeOptions): CliRuntime;
|
|
1
|
+
import { CliRuntime, CreateCliRuntimeOptions, createCliRuntime } from "./runtime.mjs";
|
|
20
2
|
declare const main: import("citty").CommandDef<import("citty").ArgsDef>;
|
|
21
3
|
interface RunCliOptions {
|
|
22
4
|
rawArgs?: string[];
|
package/dist/cli.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { createCliRuntime, runWithCliRuntime } from "./
|
|
2
|
-
import { resolveCliEnvironment } from "./
|
|
3
|
-
import { VERSION, configureColor, showSplash, withConfiguredOutput } from "./
|
|
4
|
-
import { applyProfileFromCli } from "./
|
|
5
|
-
import { loadEnvFromCwd } from "./
|
|
1
|
+
import { createCliRuntime, runWithCliRuntime } from "./runtime.mjs";
|
|
2
|
+
import { resolveCliEnvironment } from "./environment.mjs";
|
|
3
|
+
import { VERSION, configureColor, showSplash, withConfiguredOutput } from "./utils.mjs";
|
|
4
|
+
import { applyProfileFromCli } from "./commands/profile.mjs";
|
|
5
|
+
import { loadEnvFromCwd } from "./env-file.mjs";
|
|
6
6
|
import process from "node:process";
|
|
7
7
|
import { defineCommand, runMain } from "citty";
|
|
8
8
|
function shouldShowSplash(rawArgs) {
|
|
@@ -62,26 +62,26 @@ const main = defineCommand({
|
|
|
62
62
|
description: "Google Search Console Data Extractor"
|
|
63
63
|
},
|
|
64
64
|
subCommands: {
|
|
65
|
-
init: () => import("./
|
|
66
|
-
dump: () => import("./
|
|
67
|
-
query: () => import("./
|
|
68
|
-
sites: () => import("./
|
|
69
|
-
sitemaps: () => import("./
|
|
70
|
-
sync: () => import("./
|
|
71
|
-
store: () => import("./
|
|
72
|
-
inspect: () => import("./
|
|
73
|
-
indexing: () => import("./
|
|
74
|
-
entities: () => import("./
|
|
75
|
-
analyze: () => import("./
|
|
76
|
-
report: () => import("./
|
|
77
|
-
auth: () => import("./
|
|
78
|
-
login: () => import("./
|
|
79
|
-
logout: () => import("./
|
|
80
|
-
status: () => import("./
|
|
81
|
-
config: () => import("./
|
|
82
|
-
profile: () => import("./
|
|
83
|
-
doctor: () => import("./
|
|
84
|
-
mcp: () => import("./
|
|
65
|
+
init: () => import("./commands/init.mjs").then((m) => m.initCommand),
|
|
66
|
+
dump: () => import("./commands/dump.mjs").then((m) => m.dumpCommand),
|
|
67
|
+
query: () => import("./commands/query.mjs").then((m) => m.queryCommand),
|
|
68
|
+
sites: () => import("./commands/sites.mjs").then((m) => m.sitesCommand),
|
|
69
|
+
sitemaps: () => import("./commands/sitemaps.mjs").then((m) => m.sitemapsCommand),
|
|
70
|
+
sync: () => import("./commands/sync.mjs").then((m) => m.syncCommand),
|
|
71
|
+
store: () => import("./commands/store.mjs").then((m) => m.storeCommand),
|
|
72
|
+
inspect: () => import("./commands/inspect.mjs").then((m) => m.inspectCommand),
|
|
73
|
+
indexing: () => import("./commands/indexing.mjs").then((m) => m.indexingCommand),
|
|
74
|
+
entities: () => import("./commands/entities.mjs").then((m) => m.entitiesCommand),
|
|
75
|
+
analyze: () => import("./commands/analyze.mjs").then((m) => m.analyzeCommand),
|
|
76
|
+
report: () => import("./commands/report.mjs").then((m) => m.reportCommand),
|
|
77
|
+
auth: () => import("./commands/auth.mjs").then((m) => m.authCommand),
|
|
78
|
+
login: () => import("./commands/auth.mjs").then((m) => m.loginCommand),
|
|
79
|
+
logout: () => import("./commands/auth.mjs").then((m) => m.logoutCommand),
|
|
80
|
+
status: () => import("./commands/auth.mjs").then((m) => m.statusCommand),
|
|
81
|
+
config: () => import("./commands/config.mjs").then((m) => m.configCommand),
|
|
82
|
+
profile: () => import("./commands/profile.mjs").then((m) => m.profileCommand),
|
|
83
|
+
doctor: () => import("./commands/doctor.mjs").then((m) => m.doctorCommand),
|
|
84
|
+
mcp: () => import("./commands/mcp.mjs").then((m) => m.mcpCommand)
|
|
85
85
|
},
|
|
86
86
|
setup({ rawArgs }) {
|
|
87
87
|
if (shouldShowSplash(rawArgs)) showSplash();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { logger, toCSV } from "
|
|
2
|
-
import { gscErrorHandler } from "
|
|
3
|
-
import { resolveAnalysisSource } from "
|
|
1
|
+
import { logger, toCSV } from "../utils.mjs";
|
|
2
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
3
|
+
import { resolveAnalysisSource } from "../analysis-local.mjs";
|
|
4
4
|
import { defineCommand } from "citty";
|
|
5
5
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
6
6
|
const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { loadConfig, saveConfig } from "
|
|
2
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected } from "
|
|
1
|
+
import { loadConfig, saveConfig } from "../config.mjs";
|
|
2
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, noSubcommandSelected } from "../utils.mjs";
|
|
3
3
|
import { adoptCurrentConfigAsProfile, profileNameFromEmail, resolveActiveProfile } from "./profile.mjs";
|
|
4
|
-
import { authenticate, clearTokens, formatAuthProvenance, getAuth, getAuthCredentials, loadServiceAccount, loadTokens, resolveBYOK, saveTokens } from "
|
|
4
|
+
import { authenticate, clearTokens, formatAuthProvenance, getAuth, getAuthCredentials, loadServiceAccount, loadTokens, resolveBYOK, saveTokens } from "../auth.mjs";
|
|
5
5
|
import { runSmokeTest } from "./init.mjs";
|
|
6
|
+
import path from "node:path";
|
|
6
7
|
import process from "node:process";
|
|
7
8
|
import { defineCommand } from "citty";
|
|
8
|
-
import path from "node:path";
|
|
9
9
|
import { hasGscWriteScope, hasIndexingScope } from "gscdump";
|
|
10
10
|
import { ofetch } from "ofetch";
|
|
11
11
|
const AUTH_SUBCOMMANDS = [
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { allTables } from "../local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "../context.mjs";
|
|
4
|
+
import { defineCommand } from "citty";
|
|
5
|
+
import { inferLegacyTier } from "@gscdump/engine";
|
|
6
|
+
const compactCommand = defineCommand({
|
|
7
|
+
meta: {
|
|
8
|
+
name: "compact",
|
|
9
|
+
description: "Run tiered compaction (raw→d7 at 7d, d7→d30 at 30d, d30→d90 at 90d)"
|
|
10
|
+
},
|
|
11
|
+
args: {
|
|
12
|
+
"site": {
|
|
13
|
+
type: "string",
|
|
14
|
+
alias: "s",
|
|
15
|
+
description: "Restrict to a single site (default: all sites with local data)"
|
|
16
|
+
},
|
|
17
|
+
"raw-days": {
|
|
18
|
+
type: "string",
|
|
19
|
+
description: "Override raw→d7 age threshold in days (default: 7)"
|
|
20
|
+
},
|
|
21
|
+
"d7-days": {
|
|
22
|
+
type: "string",
|
|
23
|
+
description: "Override d7→d30 age threshold in days (default: 30)"
|
|
24
|
+
},
|
|
25
|
+
"d30-days": {
|
|
26
|
+
type: "string",
|
|
27
|
+
description: "Override d30→d90 age threshold in days (default: 90)"
|
|
28
|
+
},
|
|
29
|
+
"dry-run": {
|
|
30
|
+
type: "boolean",
|
|
31
|
+
default: false,
|
|
32
|
+
description: "Report tier counts per (table, site) without compacting"
|
|
33
|
+
},
|
|
34
|
+
...OUTPUT_ARGS
|
|
35
|
+
},
|
|
36
|
+
async run({ args }) {
|
|
37
|
+
const { json } = applyOutputMode(args);
|
|
38
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
39
|
+
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
40
|
+
const dryRun = Boolean(args["dry-run"]);
|
|
41
|
+
const thresholds = {};
|
|
42
|
+
if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
|
|
43
|
+
if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
|
|
44
|
+
if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
|
|
45
|
+
const liveEntries = await store.engine.listLive({
|
|
46
|
+
userId: store.userId,
|
|
47
|
+
siteId
|
|
48
|
+
});
|
|
49
|
+
if (dryRun) {
|
|
50
|
+
const report = [];
|
|
51
|
+
for (const table of allTables()) {
|
|
52
|
+
const bySite = groupBySite(liveEntries.filter((entry) => entry.table === table));
|
|
53
|
+
for (const [s, group] of bySite) report.push({
|
|
54
|
+
table,
|
|
55
|
+
siteId: s,
|
|
56
|
+
...countByTier(group)
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (json) {
|
|
60
|
+
console.log(JSON.stringify({
|
|
61
|
+
thresholds,
|
|
62
|
+
plan: report
|
|
63
|
+
}, null, 2));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
console.log();
|
|
67
|
+
console.log(` table site raw d7 d30 d90`);
|
|
68
|
+
for (const r of report) console.log(` ${r.table.padEnd(20)} ${(r.siteId ?? "-").padEnd(20)} ${String(r.raw).padStart(4)} ${String(r.d7).padStart(4)} ${String(r.d30).padStart(4)} ${String(r.d90).padStart(4)}`);
|
|
69
|
+
console.log();
|
|
70
|
+
logger.info(`compact --dry-run: ${report.length} (table, site) pair(s) — pass without --dry-run to apply`);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const summary = [];
|
|
74
|
+
for (const table of allTables()) {
|
|
75
|
+
const entries = liveEntries.filter((entry) => entry.table === table);
|
|
76
|
+
const siteIds = new Set(entries.map((e) => e.siteId));
|
|
77
|
+
for (const targetSite of siteIds) {
|
|
78
|
+
logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
|
|
79
|
+
await store.engine.compactTiered({
|
|
80
|
+
userId: store.userId,
|
|
81
|
+
siteId: targetSite,
|
|
82
|
+
table
|
|
83
|
+
}, thresholds);
|
|
84
|
+
summary.push({
|
|
85
|
+
table,
|
|
86
|
+
siteId: targetSite
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (json) {
|
|
91
|
+
console.log(JSON.stringify({
|
|
92
|
+
thresholds,
|
|
93
|
+
compacted: summary
|
|
94
|
+
}, null, 2));
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
logger.success(`compact: done`);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
function groupBySite(entries) {
|
|
101
|
+
const m = /* @__PURE__ */ new Map();
|
|
102
|
+
for (const e of entries) {
|
|
103
|
+
const arr = m.get(e.siteId) ?? [];
|
|
104
|
+
arr.push(e);
|
|
105
|
+
m.set(e.siteId, arr);
|
|
106
|
+
}
|
|
107
|
+
return m;
|
|
108
|
+
}
|
|
109
|
+
function countByTier(entries) {
|
|
110
|
+
let raw = 0;
|
|
111
|
+
let d7 = 0;
|
|
112
|
+
let d30 = 0;
|
|
113
|
+
let d90 = 0;
|
|
114
|
+
for (const e of entries) {
|
|
115
|
+
const tier = e.tier ?? inferLegacyTier(e) ?? "raw";
|
|
116
|
+
if (tier === "raw") raw++;
|
|
117
|
+
else if (tier === "d7") d7++;
|
|
118
|
+
else if (tier === "d30") d30++;
|
|
119
|
+
else if (tier === "d90") d90++;
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
raw,
|
|
123
|
+
d7,
|
|
124
|
+
d30,
|
|
125
|
+
d90
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
export { compactCommand };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { getConfigPath, loadConfig, saveConfig } from "
|
|
2
|
-
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger, noSubcommandSelected } from "
|
|
1
|
+
import { getConfigPath, loadConfig, saveConfig } from "../config.mjs";
|
|
2
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger, noSubcommandSelected } from "../utils.mjs";
|
|
3
3
|
import process from "node:process";
|
|
4
4
|
import { defineCommand } from "citty";
|
|
5
5
|
const showCommand = defineCommand({
|
|
@@ -126,7 +126,7 @@ const configCommand = defineCommand({
|
|
|
126
126
|
args: { ...OUTPUT_ARGS },
|
|
127
127
|
async run({ args }) {
|
|
128
128
|
const { json } = applyOutputMode(args);
|
|
129
|
-
const { resolveDataDir } = await import("
|
|
129
|
+
const { resolveDataDir } = await import("../config.mjs");
|
|
130
130
|
const fs = await import("node:fs/promises");
|
|
131
131
|
const config = await loadConfig();
|
|
132
132
|
const issues = [];
|
|
@@ -151,7 +151,7 @@ const configCommand = defineCommand({
|
|
|
151
151
|
message: `${dataDirDisplay} does not exist (will be created on first sync)`
|
|
152
152
|
});
|
|
153
153
|
if (config.defaultSite) if (!!config.clientId && !!config.clientSecret) {
|
|
154
|
-
const { createCommandContext } = await import("
|
|
154
|
+
const { createCommandContext } = await import("../context.mjs");
|
|
155
155
|
const ctx = await createCommandContext({ needsAuth: true }).catch(() => null);
|
|
156
156
|
if (ctx) {
|
|
157
157
|
const sites = await ctx.loadSites().catch(() => null);
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { loadConfig } from "
|
|
2
|
-
import { resolveCliEnvironment } from "
|
|
3
|
-
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "
|
|
4
|
-
import { parseEnvFile } from "
|
|
5
|
-
import { loadTokens, resolveAuth, resolveBYOK } from "
|
|
6
|
-
import {
|
|
1
|
+
import { loadConfig } from "../config.mjs";
|
|
2
|
+
import { resolveCliEnvironment } from "../environment.mjs";
|
|
3
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "../utils.mjs";
|
|
4
|
+
import { parseEnvFile } from "../env-file.mjs";
|
|
5
|
+
import { loadTokens, resolveAuth, resolveBYOK } from "../auth.mjs";
|
|
6
|
+
import { createLocalStore } from "../local-store.mjs";
|
|
7
|
+
import { createCommandContext } from "../context.mjs";
|
|
8
|
+
import path from "node:path";
|
|
7
9
|
import process from "node:process";
|
|
8
10
|
import { defineCommand } from "citty";
|
|
9
11
|
import fs from "node:fs/promises";
|
|
10
|
-
import path from "node:path";
|
|
11
12
|
import { googleSearchConsole, hasGscWriteScope, hasIndexingScope } from "gscdump";
|
|
12
13
|
import { ofetch } from "ofetch";
|
|
13
14
|
const REQUIRED_SCOPES = [
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, runWithConcurrency, toCSV } from "
|
|
2
|
-
import { allTables
|
|
3
|
-
import {
|
|
1
|
+
import { ALL_SEARCH_TYPES, OUTPUT_ARGS, applyOutputMode, displayPath, logger, parseSearchType, runWithConcurrency, toCSV } from "../utils.mjs";
|
|
2
|
+
import { allTables } from "../local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "../context.mjs";
|
|
4
|
+
import { readParquetRows } from "../native-duckdb.mjs";
|
|
5
|
+
import path from "node:path";
|
|
4
6
|
import process from "node:process";
|
|
5
7
|
import { defineCommand } from "citty";
|
|
6
8
|
import fs from "node:fs/promises";
|
|
7
|
-
import path from "node:path";
|
|
8
9
|
const DEFAULT_OUT = "./gscdump-export";
|
|
9
10
|
const FORMATS = [
|
|
10
11
|
"parquet",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, progressBar, runWithConcurrency } from "
|
|
2
|
-
import { createCommandContext } from "
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, progressBar, runWithConcurrency } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
3
|
import process from "node:process";
|
|
4
4
|
import { defineCommand } from "citty";
|
|
5
5
|
import { readFile } from "node:fs/promises";
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath } from "../utils.mjs";
|
|
2
|
+
import { allTables } from "../local-store.mjs";
|
|
3
|
+
import { createCommandContext } from "../context.mjs";
|
|
4
|
+
import { materializeParquetTables } from "../native-duckdb.mjs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { defineCommand } from "citty";
|
|
7
|
+
async function exportToDuckDB(opts) {
|
|
8
|
+
const outPath = path.resolve(opts.outPath);
|
|
9
|
+
const entries = await opts.engine.listLive({
|
|
10
|
+
userId: opts.userId,
|
|
11
|
+
siteId: opts.siteId
|
|
12
|
+
});
|
|
13
|
+
const inputs = [];
|
|
14
|
+
for (const table of allTables()) {
|
|
15
|
+
const tableEntries = entries.filter((entry) => entry.table === table);
|
|
16
|
+
if (tableEntries.length > 0) inputs.push({
|
|
17
|
+
table,
|
|
18
|
+
filePaths: tableEntries.map((entry) => path.join(opts.dataDir, entry.objectKey))
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
const tables = await materializeParquetTables(outPath, inputs, opts.force);
|
|
22
|
+
return {
|
|
23
|
+
outPath,
|
|
24
|
+
tables,
|
|
25
|
+
totalRows: tables.reduce((acc, t) => acc + t.rows, 0)
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const exportCommand = defineCommand({
|
|
29
|
+
meta: {
|
|
30
|
+
name: "export",
|
|
31
|
+
description: "Pack live Parquet partitions into a single .duckdb file for portable distribution (browser attach, CDN serving, etc.)"
|
|
32
|
+
},
|
|
33
|
+
args: {
|
|
34
|
+
out: {
|
|
35
|
+
type: "string",
|
|
36
|
+
required: true,
|
|
37
|
+
description: "Output path for the .duckdb file"
|
|
38
|
+
},
|
|
39
|
+
site: {
|
|
40
|
+
type: "string",
|
|
41
|
+
description: "Limit export to a single site URL (omit to include all)"
|
|
42
|
+
},
|
|
43
|
+
force: {
|
|
44
|
+
type: "boolean",
|
|
45
|
+
default: false,
|
|
46
|
+
description: "Overwrite the output file if it already exists"
|
|
47
|
+
},
|
|
48
|
+
...OUTPUT_ARGS
|
|
49
|
+
},
|
|
50
|
+
async run({ args }) {
|
|
51
|
+
const { json } = applyOutputMode(args);
|
|
52
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
53
|
+
const siteId = args.site ? store.siteIdFor(args.site) : void 0;
|
|
54
|
+
const result = await exportToDuckDB({
|
|
55
|
+
engine: store.engine,
|
|
56
|
+
dataDir: store.dataDir,
|
|
57
|
+
userId: store.userId,
|
|
58
|
+
siteId,
|
|
59
|
+
outPath: args.out,
|
|
60
|
+
force: args.force
|
|
61
|
+
});
|
|
62
|
+
if (json) {
|
|
63
|
+
console.log(JSON.stringify(result, null, 2));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (result.tables.length === 0) {
|
|
67
|
+
console.log(`\n No data to export. Run \`gscdump sync\` first.`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
for (const t of result.tables) console.log(` ${t.table.padEnd(15)} ${String(t.files).padStart(4)} parquet → ${t.table} (${t.rows.toLocaleString()} rows)`);
|
|
71
|
+
console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${displayPath(result.outPath)}`);
|
|
72
|
+
console.log(`\n Attach from DuckDB: \x1B[36mATTACH '${result.outPath}' AS gsc (READ_ONLY); SELECT * FROM gsc.pages LIMIT 10;\x1B[0m`);
|
|
73
|
+
console.log(` Attach in a browser: use DuckDB-WASM registerFileBuffer + \x1B[36mATTACH 'gsc.duckdb' AS gsc (READ_ONLY)\x1B[0m`);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
export { exportCommand, exportToDuckDB };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { defineCommand } from "citty";
|
|
4
|
+
const DEFAULT_GRACE_HOURS = 24;
|
|
5
|
+
const gcCommand = defineCommand({
|
|
6
|
+
meta: {
|
|
7
|
+
name: "gc",
|
|
8
|
+
description: "Delete orphaned object-store files not referenced by any manifest entry"
|
|
9
|
+
},
|
|
10
|
+
args: {
|
|
11
|
+
"grace-hours": {
|
|
12
|
+
type: "string",
|
|
13
|
+
default: String(DEFAULT_GRACE_HOURS),
|
|
14
|
+
description: `Spare orphans younger than this (default: ${DEFAULT_GRACE_HOURS}h)`
|
|
15
|
+
},
|
|
16
|
+
"site": {
|
|
17
|
+
type: "string",
|
|
18
|
+
alias: "s",
|
|
19
|
+
description: "Restrict to a single site (default: all sites)"
|
|
20
|
+
},
|
|
21
|
+
"dry-run": {
|
|
22
|
+
type: "boolean",
|
|
23
|
+
default: false,
|
|
24
|
+
description: "List retired manifest entries past the grace window without deleting"
|
|
25
|
+
},
|
|
26
|
+
...OUTPUT_ARGS
|
|
27
|
+
},
|
|
28
|
+
async run({ args }) {
|
|
29
|
+
const { json } = applyOutputMode(args);
|
|
30
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
31
|
+
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
32
|
+
const graceMs = Number(args["grace-hours"]) * 36e5;
|
|
33
|
+
if (args["dry-run"]) {
|
|
34
|
+
const cutoff = Date.now() - graceMs;
|
|
35
|
+
const candidates = [];
|
|
36
|
+
const all = await store.engine.listAll({
|
|
37
|
+
userId: store.userId,
|
|
38
|
+
siteId
|
|
39
|
+
});
|
|
40
|
+
for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
|
|
41
|
+
table: e.table,
|
|
42
|
+
siteId: e.siteId,
|
|
43
|
+
partition: e.partition,
|
|
44
|
+
retiredAt: e.retiredAt,
|
|
45
|
+
objectKey: e.objectKey
|
|
46
|
+
});
|
|
47
|
+
if (json) {
|
|
48
|
+
console.log(JSON.stringify({
|
|
49
|
+
graceHours: Number(args["grace-hours"]),
|
|
50
|
+
candidates
|
|
51
|
+
}, null, 2));
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
console.log();
|
|
55
|
+
for (const c of candidates) console.log(` ${c.objectKey} \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
|
|
56
|
+
console.log();
|
|
57
|
+
logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const result = await store.engine.gcOrphans({
|
|
61
|
+
userId: store.userId,
|
|
62
|
+
siteId
|
|
63
|
+
}, graceMs);
|
|
64
|
+
if (json) {
|
|
65
|
+
console.log(JSON.stringify({
|
|
66
|
+
graceHours: Number(args["grace-hours"]),
|
|
67
|
+
deleted: result.deleted
|
|
68
|
+
}, null, 2));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
export { gcCommand };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, readUrlList } from "
|
|
2
|
-
import { createCommandContext } from "
|
|
3
|
-
import { gscErrorHandler } from "
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, readUrlList } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
4
4
|
import process from "node:process";
|
|
5
5
|
import { defineCommand } from "citty";
|
|
6
6
|
import { batchRequestIndexing, fetchSitemapUrls, getIndexingMetadata, requestIndexing, runSequentialBatch } from "gscdump";
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { defaultDataDir, loadConfig, saveConfig } from "
|
|
2
|
-
import { applyCliEnvironment } from "
|
|
3
|
-
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "
|
|
4
|
-
import { authenticate, getAuthCredentials, loadTokens, resolveBYOK, saveTokens } from "
|
|
1
|
+
import { defaultDataDir, loadConfig, saveConfig } from "../config.mjs";
|
|
2
|
+
import { applyCliEnvironment } from "../environment.mjs";
|
|
3
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger } from "../utils.mjs";
|
|
4
|
+
import { authenticate, getAuthCredentials, loadTokens, resolveBYOK, saveTokens } from "../auth.mjs";
|
|
5
|
+
import path from "node:path";
|
|
5
6
|
import process from "node:process";
|
|
6
7
|
import { defineCommand } from "citty";
|
|
7
8
|
import fs from "node:fs/promises";
|
|
8
|
-
import path from "node:path";
|
|
9
9
|
import { confirm, isCancel, text } from "@clack/prompts";
|
|
10
10
|
import { googleSearchConsole } from "gscdump";
|
|
11
11
|
const ENV_LINE_RE = /^([^=]+)=(.*)$/;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { OUTPUT_ARGS, applyOutputMode, logger, readUrlList } from "
|
|
2
|
-
import { createCommandContext } from "
|
|
3
|
-
import { gscErrorHandler } from "
|
|
1
|
+
import { OUTPUT_ARGS, applyOutputMode, logger, readUrlList } from "../utils.mjs";
|
|
2
|
+
import { createCommandContext } from "../context.mjs";
|
|
3
|
+
import { gscErrorHandler } from "../error-handler.mjs";
|
|
4
4
|
import process from "node:process";
|
|
5
5
|
import { defineCommand } from "citty";
|
|
6
6
|
import { batchInspectUrls, fetchSitemapUrls } from "gscdump";
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { loadConfig } from "../config.mjs";
|
|
2
|
+
import { VERSION } from "../utils.mjs";
|
|
3
|
+
import { loadTokens, resolveAuth, resolveBYOK, resolveServiceAccount } from "../auth.mjs";
|
|
4
|
+
import { createGscMcpServer } from "../mcp/server/index.mjs";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
import { defineCommand } from "citty";
|
|
7
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
8
|
+
async function checkAuth() {
|
|
9
|
+
if (await resolveServiceAccount().then(Boolean).catch(() => false)) return { ok: true };
|
|
10
|
+
if (resolveBYOK()) return { ok: true };
|
|
11
|
+
const config = await loadConfig();
|
|
12
|
+
if (!config.clientId && !config.clientSecret) return {
|
|
13
|
+
ok: false,
|
|
14
|
+
error: `GSCDump not configured.
|
|
15
|
+
|
|
16
|
+
Run this command to set up authentication:
|
|
17
|
+
|
|
18
|
+
npx @gscdump/cli init
|
|
19
|
+
|
|
20
|
+
Or set BYOK env vars: GSC_ACCESS_TOKEN, or GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN
|
|
21
|
+
(GOOGLE_* aliases also accepted).
|
|
22
|
+
|
|
23
|
+
Then restart your MCP client.`
|
|
24
|
+
};
|
|
25
|
+
if (!await loadTokens()) return {
|
|
26
|
+
ok: false,
|
|
27
|
+
error: `Authentication missing.
|
|
28
|
+
|
|
29
|
+
Run this command to authenticate:
|
|
30
|
+
|
|
31
|
+
npx @gscdump/cli auth login
|
|
32
|
+
|
|
33
|
+
Then restart your MCP client.`
|
|
34
|
+
};
|
|
35
|
+
return { ok: true };
|
|
36
|
+
}
|
|
37
|
+
const mcpCommand = defineCommand({
|
|
38
|
+
meta: {
|
|
39
|
+
name: "mcp",
|
|
40
|
+
description: "Start MCP server for AI assistants"
|
|
41
|
+
},
|
|
42
|
+
async run() {
|
|
43
|
+
const authCheck = await checkAuth();
|
|
44
|
+
if (!authCheck.ok) {
|
|
45
|
+
process.stderr.write(`\n${authCheck.error}\n\n`);
|
|
46
|
+
process.exit(1);
|
|
47
|
+
}
|
|
48
|
+
const server = createGscMcpServer({
|
|
49
|
+
name: "gscdump",
|
|
50
|
+
version: VERSION,
|
|
51
|
+
getAuth: () => resolveAuth({ interactive: false })
|
|
52
|
+
});
|
|
53
|
+
const transport = new StdioServerTransport();
|
|
54
|
+
await server.connect(transport);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
export { mcpCommand };
|
|
@@ -1,24 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { useCliRuntime } from "../runtime.mjs";
|
|
2
|
+
import { getConfigDir, setConfigDir } from "../config.mjs";
|
|
3
|
+
import { resolveCliEnvironment } from "../environment.mjs";
|
|
4
|
+
import { OUTPUT_ARGS, applyOutputMode, displayPath, logger, noSubcommandSelected } from "../utils.mjs";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
4
7
|
import process from "node:process";
|
|
5
8
|
import { defineCommand } from "citty";
|
|
6
9
|
import fs from "node:fs";
|
|
7
10
|
import fs$1 from "node:fs/promises";
|
|
8
|
-
import os from "node:os";
|
|
9
|
-
import path from "node:path";
|
|
10
11
|
import { confirm, isCancel } from "@clack/prompts";
|
|
11
|
-
var profile_exports = /* @__PURE__ */ __exportAll({
|
|
12
|
-
adoptCurrentConfigAsProfile: () => adoptCurrentConfigAsProfile,
|
|
13
|
-
applyProfileFromCli: () => applyProfileFromCli,
|
|
14
|
-
createProfile: () => createProfile,
|
|
15
|
-
getProfileDir: () => getProfileDir,
|
|
16
|
-
listProfiles: () => listProfiles,
|
|
17
|
-
profileCommand: () => profileCommand,
|
|
18
|
-
profileNameFromEmail: () => profileNameFromEmail,
|
|
19
|
-
resolveActiveProfile: () => resolveActiveProfile,
|
|
20
|
-
setActiveProfile: () => setActiveProfile
|
|
21
|
-
});
|
|
22
12
|
const ROOT_DIR = path.join(os.homedir(), ".config", "gscdump");
|
|
23
13
|
const PROFILES_DIR = path.join(ROOT_DIR, "profiles");
|
|
24
14
|
const ACTIVE_MARKER = path.join(ROOT_DIR, "active-profile");
|
|
@@ -287,4 +277,4 @@ const profileCommand = defineCommand({
|
|
|
287
277
|
});
|
|
288
278
|
}
|
|
289
279
|
});
|
|
290
|
-
export { adoptCurrentConfigAsProfile, applyProfileFromCli,
|
|
280
|
+
export { adoptCurrentConfigAsProfile, applyProfileFromCli, createProfile, getProfileDir, listProfiles, profileCommand, profileNameFromEmail, resolveActiveProfile, setActiveProfile };
|