@pagebridge/cli 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-check-types.log +4 -0
- package/dist/commands/diagnose.d.ts.map +1 -1
- package/dist/commands/diagnose.js +43 -25
- package/dist/commands/list-sites.d.ts.map +1 -1
- package/dist/commands/list-sites.js +22 -17
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +128 -70
- package/dist/index.js +0 -6
- package/dist/logger.d.ts +7 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +19 -0
- package/dist/migrate.d.ts +2 -0
- package/dist/migrate.d.ts.map +1 -0
- package/dist/migrate.js +13 -0
- package/dist/resolve-config.d.ts +17 -0
- package/dist/resolve-config.d.ts.map +1 -0
- package/dist/resolve-config.js +21 -0
- package/package.json +3 -5
- package/src/commands/diagnose.ts +45 -25
- package/src/commands/list-sites.ts +24 -20
- package/src/commands/sync.ts +137 -72
- package/src/index.ts +0 -8
- package/src/logger.ts +22 -0
- package/src/migrate.ts +13 -0
- package/src/resolve-config.ts +32 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnose.d.ts","sourceRoot":"","sources":["../../src/commands/diagnose.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"diagnose.d.ts","sourceRoot":"","sources":["../../src/commands/diagnose.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,eAAO,MAAM,eAAe,SA2GxB,CAAC"}
|
|
@@ -1,19 +1,32 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import { createDb, unmatchDiagnostics, eq, desc, } from "@pagebridge/db";
|
|
3
|
+
import { resolve, requireConfig } from "../resolve-config.js";
|
|
4
|
+
import { log } from "../logger.js";
|
|
5
|
+
import { migrateIfRequested } from "../migrate.js";
|
|
4
6
|
export const diagnoseCommand = new Command("diagnose")
|
|
5
7
|
.description("View diagnostics for unmatched URLs")
|
|
6
8
|
.requiredOption("--site <url>", "GSC site URL (e.g., sc-domain:example.com)")
|
|
7
9
|
.option("--reason <reason>", "Filter by unmatch reason")
|
|
8
10
|
.option("--limit <n>", "Limit number of results", "20")
|
|
9
11
|
.option("--json", "Output as JSON")
|
|
12
|
+
.option("--migrate", "Run database migrations before querying")
|
|
13
|
+
.option("--db-url <url>", "PostgreSQL connection string")
|
|
10
14
|
.action(async (options) => {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
const dbUrl = resolve(options.dbUrl, "DATABASE_URL");
|
|
16
|
+
requireConfig([
|
|
17
|
+
{ name: "DATABASE_URL", flag: "--db-url <url>", envVar: "DATABASE_URL", value: dbUrl },
|
|
18
|
+
]);
|
|
19
|
+
// Run migrations if requested
|
|
20
|
+
await migrateIfRequested(!!options.migrate, dbUrl);
|
|
21
|
+
const { db, close } = createDb(dbUrl);
|
|
22
|
+
// Register shutdown handlers
|
|
23
|
+
const shutdown = async () => {
|
|
24
|
+
log.warn("Received shutdown signal, closing connections...");
|
|
25
|
+
await close();
|
|
26
|
+
process.exit(130);
|
|
27
|
+
};
|
|
28
|
+
process.on("SIGTERM", shutdown);
|
|
29
|
+
process.on("SIGINT", shutdown);
|
|
17
30
|
try {
|
|
18
31
|
const query = db
|
|
19
32
|
.select()
|
|
@@ -27,8 +40,8 @@ export const diagnoseCommand = new Command("diagnose")
|
|
|
27
40
|
return;
|
|
28
41
|
}
|
|
29
42
|
if (results.length === 0) {
|
|
30
|
-
|
|
31
|
-
|
|
43
|
+
log.info(`No unmatched URLs found for ${options.site}`);
|
|
44
|
+
log.info(`Run 'sync --site ${options.site}' first to generate diagnostics.`);
|
|
32
45
|
return;
|
|
33
46
|
}
|
|
34
47
|
// Group by reason
|
|
@@ -38,23 +51,22 @@ export const diagnoseCommand = new Command("diagnose")
|
|
|
38
51
|
existing.push(r);
|
|
39
52
|
byReason.set(r.unmatchReason, existing);
|
|
40
53
|
}
|
|
41
|
-
|
|
42
|
-
|
|
54
|
+
log.info(`\nUnmatched URL Diagnostics for ${options.site}\n`);
|
|
55
|
+
log.info(`Total: ${results.length} unmatched URLs\n`);
|
|
43
56
|
for (const [reason, items] of byReason) {
|
|
44
|
-
|
|
45
|
-
console.log();
|
|
57
|
+
log.info(`${getReasonEmoji(reason)} ${getReasonDescription(reason)} (${items.length}):`);
|
|
46
58
|
for (const item of items) {
|
|
47
|
-
|
|
59
|
+
log.info(` ${item.gscUrl}`);
|
|
48
60
|
if (item.extractedSlug) {
|
|
49
|
-
|
|
61
|
+
log.info(` Extracted slug: "${item.extractedSlug}"`);
|
|
50
62
|
}
|
|
51
63
|
if (item.similarSlugs) {
|
|
52
64
|
try {
|
|
53
65
|
const similar = JSON.parse(item.similarSlugs);
|
|
54
66
|
if (similar.length > 0) {
|
|
55
|
-
|
|
67
|
+
log.info(` Similar slugs in Sanity:`);
|
|
56
68
|
for (const s of similar) {
|
|
57
|
-
|
|
69
|
+
log.info(` - ${s}`);
|
|
58
70
|
}
|
|
59
71
|
}
|
|
60
72
|
}
|
|
@@ -62,17 +74,23 @@ export const diagnoseCommand = new Command("diagnose")
|
|
|
62
74
|
// Ignore parse errors
|
|
63
75
|
}
|
|
64
76
|
}
|
|
65
|
-
console.log();
|
|
66
77
|
}
|
|
67
78
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
79
|
+
log.info(`\nTo fix unmatched URLs:`);
|
|
80
|
+
log.info(` 1. Check if the Sanity document exists with the correct slug`);
|
|
81
|
+
log.info(` 2. Verify the document type is in the contentTypes list`);
|
|
82
|
+
log.info(` 3. Ensure the slug field name matches your configuration`);
|
|
83
|
+
log.info(` 4. If using a path prefix, verify it matches your URL structure`);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
87
|
+
log.error(`Diagnose failed: ${message}`);
|
|
88
|
+
process.exitCode = 1;
|
|
73
89
|
}
|
|
74
90
|
finally {
|
|
75
|
-
await
|
|
91
|
+
await close();
|
|
92
|
+
process.removeListener("SIGTERM", shutdown);
|
|
93
|
+
process.removeListener("SIGINT", shutdown);
|
|
76
94
|
}
|
|
77
95
|
});
|
|
78
96
|
function getReasonEmoji(reason) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"list-sites.d.ts","sourceRoot":"","sources":["../../src/commands/list-sites.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"list-sites.d.ts","sourceRoot":"","sources":["../../src/commands/list-sites.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,eAAO,MAAM,gBAAgB,SA6CzB,CAAC"}
|
|
@@ -1,38 +1,43 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { GSCClient } from "@pagebridge/core";
|
|
3
|
+
import { resolve, requireConfig } from "../resolve-config.js";
|
|
4
|
+
import { log } from "../logger.js";
|
|
3
5
|
export const listSitesCommand = new Command("list-sites")
|
|
4
6
|
.description("List all sites the service account has access to")
|
|
5
|
-
.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
.option("--google-service-account <json>", "Google service account JSON")
|
|
8
|
+
.action(async (options) => {
|
|
9
|
+
const googleServiceAccount = resolve(options.googleServiceAccount, "GOOGLE_SERVICE_ACCOUNT");
|
|
10
|
+
requireConfig([
|
|
11
|
+
{ name: "GOOGLE_SERVICE_ACCOUNT", flag: "--google-service-account <json>", envVar: "GOOGLE_SERVICE_ACCOUNT", value: googleServiceAccount },
|
|
12
|
+
]);
|
|
10
13
|
let credentials;
|
|
11
14
|
try {
|
|
12
|
-
credentials = JSON.parse(
|
|
15
|
+
credentials = JSON.parse(googleServiceAccount);
|
|
13
16
|
}
|
|
14
17
|
catch {
|
|
15
|
-
|
|
18
|
+
log.error("Failed to parse GOOGLE_SERVICE_ACCOUNT as JSON");
|
|
16
19
|
process.exit(1);
|
|
17
20
|
}
|
|
18
|
-
|
|
21
|
+
log.info(`Using service account: ${credentials.client_email}`);
|
|
19
22
|
const gsc = new GSCClient({ credentials });
|
|
20
23
|
try {
|
|
21
24
|
const sites = await gsc.listSites();
|
|
22
25
|
if (sites.length === 0) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
log.warn("No sites found. The service account has no access to any GSC properties.");
|
|
27
|
+
log.info("\nTo fix this:");
|
|
28
|
+
log.info("1. Go to Google Search Console > Settings > Users and permissions");
|
|
29
|
+
log.info(`2. Add user: ${credentials.client_email}`);
|
|
30
|
+
log.info("3. Set permission level to 'Full'");
|
|
28
31
|
}
|
|
29
32
|
else {
|
|
30
|
-
|
|
31
|
-
sites.forEach((site) =>
|
|
32
|
-
|
|
33
|
+
log.info(`Found ${sites.length} site(s):\n`);
|
|
34
|
+
sites.forEach((site) => log.info(` ${site}`));
|
|
35
|
+
log.info('\nUse one of these exact values with: pnpm sync --site "<value>"');
|
|
33
36
|
}
|
|
34
37
|
}
|
|
35
38
|
catch (error) {
|
|
36
|
-
|
|
39
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
40
|
+
log.error(`Failed to list sites: ${message}`);
|
|
41
|
+
process.exit(1);
|
|
37
42
|
}
|
|
38
43
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/commands/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/commands/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkCpC,eAAO,MAAM,WAAW,SAkYpB,CAAC"}
|
package/dist/commands/sync.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Command } from "commander";
|
|
2
2
|
import { createClient as createSanityClient } from "@sanity/client";
|
|
3
|
-
import postgres from "postgres";
|
|
4
3
|
import { GSCClient, SyncEngine, DecayDetector, URLMatcher, TaskGenerator, } from "@pagebridge/core";
|
|
5
|
-
import {
|
|
4
|
+
import { createDb, sql, unmatchDiagnostics } from "@pagebridge/db";
|
|
5
|
+
import { resolve, requireConfig } from "../resolve-config.js";
|
|
6
|
+
import { log } from "../logger.js";
|
|
7
|
+
import { migrateIfRequested } from "../migrate.js";
|
|
6
8
|
function daysAgo(days) {
|
|
7
9
|
const date = new Date();
|
|
8
10
|
date.setDate(date.getDate() - days);
|
|
@@ -14,7 +16,7 @@ function createTimer(debug) {
|
|
|
14
16
|
end: (label, startTime) => {
|
|
15
17
|
if (debug) {
|
|
16
18
|
const elapsed = ((performance.now() - startTime) / 1000).toFixed(2);
|
|
17
|
-
|
|
19
|
+
log.debug(`${label} completed in ${elapsed}s`, true);
|
|
18
20
|
}
|
|
19
21
|
},
|
|
20
22
|
};
|
|
@@ -29,38 +31,92 @@ export const syncCommand = new Command("sync")
|
|
|
29
31
|
.option("--diagnose", "Show detailed diagnostics for unmatched URLs")
|
|
30
32
|
.option("--diagnose-url <url>", "Diagnose why a specific URL is not matching")
|
|
31
33
|
.option("--debug", "Enable debug logging with timing information")
|
|
34
|
+
.option("--migrate", "Run database migrations before syncing")
|
|
35
|
+
.option("--google-service-account <json>", "Google service account JSON")
|
|
36
|
+
.option("--db-url <url>", "PostgreSQL connection string")
|
|
37
|
+
.option("--sanity-project-id <id>", "Sanity project ID")
|
|
38
|
+
.option("--sanity-dataset <name>", "Sanity dataset name")
|
|
39
|
+
.option("--sanity-token <token>", "Sanity API token")
|
|
40
|
+
.option("--site-url <url>", "Your website base URL for URL matching")
|
|
32
41
|
.action(async (options) => {
|
|
33
42
|
const timer = createTimer(options.debug);
|
|
34
43
|
const syncStartTime = timer.start();
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
"
|
|
39
|
-
|
|
40
|
-
"SANITY_TOKEN",
|
|
41
|
-
"SITE_URL",
|
|
42
|
-
];
|
|
43
|
-
for (const envVar of requiredEnvVars) {
|
|
44
|
-
if (!process.env[envVar]) {
|
|
45
|
-
console.error(`Missing required environment variable: ${envVar}`);
|
|
46
|
-
process.exit(1);
|
|
47
|
-
}
|
|
44
|
+
// Validate --quiet-period
|
|
45
|
+
const quietPeriodDays = parseInt(options.quietPeriod);
|
|
46
|
+
if (isNaN(quietPeriodDays)) {
|
|
47
|
+
log.error(`Invalid --quiet-period value: "${options.quietPeriod}". Must be a number.`);
|
|
48
|
+
process.exit(1);
|
|
48
49
|
}
|
|
50
|
+
const googleServiceAccount = resolve(options.googleServiceAccount, "GOOGLE_SERVICE_ACCOUNT");
|
|
51
|
+
const dbUrl = resolve(options.dbUrl, "DATABASE_URL");
|
|
52
|
+
const sanityProjectId = resolve(options.sanityProjectId, "SANITY_PROJECT_ID");
|
|
53
|
+
const sanityDataset = resolve(options.sanityDataset, "SANITY_DATASET");
|
|
54
|
+
const sanityToken = resolve(options.sanityToken, "SANITY_TOKEN");
|
|
55
|
+
const siteUrl = resolve(options.siteUrl, "SITE_URL");
|
|
56
|
+
requireConfig([
|
|
57
|
+
{ name: "GOOGLE_SERVICE_ACCOUNT", flag: "--google-service-account <json>", envVar: "GOOGLE_SERVICE_ACCOUNT", value: googleServiceAccount },
|
|
58
|
+
{ name: "DATABASE_URL", flag: "--db-url <url>", envVar: "DATABASE_URL", value: dbUrl },
|
|
59
|
+
{ name: "SANITY_PROJECT_ID", flag: "--sanity-project-id <id>", envVar: "SANITY_PROJECT_ID", value: sanityProjectId },
|
|
60
|
+
{ name: "SANITY_DATASET", flag: "--sanity-dataset <name>", envVar: "SANITY_DATASET", value: sanityDataset },
|
|
61
|
+
{ name: "SANITY_TOKEN", flag: "--sanity-token <token>", envVar: "SANITY_TOKEN", value: sanityToken },
|
|
62
|
+
{ name: "SITE_URL", flag: "--site-url <url>", envVar: "SITE_URL", value: siteUrl },
|
|
63
|
+
]);
|
|
64
|
+
// Run migrations if requested
|
|
65
|
+
await migrateIfRequested(!!options.migrate, dbUrl);
|
|
49
66
|
let t = timer.start();
|
|
50
67
|
const sanity = createSanityClient({
|
|
51
|
-
projectId:
|
|
52
|
-
dataset:
|
|
53
|
-
token:
|
|
68
|
+
projectId: sanityProjectId,
|
|
69
|
+
dataset: sanityDataset,
|
|
70
|
+
token: sanityToken,
|
|
54
71
|
apiVersion: "2024-01-01",
|
|
55
72
|
useCdn: false,
|
|
56
73
|
});
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
credentials
|
|
61
|
-
}
|
|
74
|
+
const { db, close } = createDb(dbUrl);
|
|
75
|
+
let credentials;
|
|
76
|
+
try {
|
|
77
|
+
credentials = JSON.parse(googleServiceAccount);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
log.error("Failed to parse GOOGLE_SERVICE_ACCOUNT as JSON");
|
|
81
|
+
await close();
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
const gsc = new GSCClient({ credentials });
|
|
62
85
|
timer.end("Client initialization", t);
|
|
63
|
-
|
|
86
|
+
// Register shutdown handlers
|
|
87
|
+
const shutdown = async () => {
|
|
88
|
+
log.warn("Received shutdown signal, closing connections...");
|
|
89
|
+
await close();
|
|
90
|
+
process.exit(130);
|
|
91
|
+
};
|
|
92
|
+
process.on("SIGTERM", shutdown);
|
|
93
|
+
process.on("SIGINT", shutdown);
|
|
94
|
+
// Pre-flight connection validation
|
|
95
|
+
try {
|
|
96
|
+
log.info("Validating connections...");
|
|
97
|
+
t = timer.start();
|
|
98
|
+
await db.execute(sql `SELECT 1`);
|
|
99
|
+
timer.end("DB connection check", t);
|
|
100
|
+
t = timer.start();
|
|
101
|
+
await sanity.fetch('*[_type == "gscSite"][0]{ _id }');
|
|
102
|
+
timer.end("Sanity connection check", t);
|
|
103
|
+
t = timer.start();
|
|
104
|
+
const sites = await gsc.listSites();
|
|
105
|
+
timer.end("GSC connection check", t);
|
|
106
|
+
if (!sites.includes(options.site)) {
|
|
107
|
+
log.warn(`Site "${options.site}" not found in GSC site list. It may be new or the service account may lack access.`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
112
|
+
log.error(`Pre-flight connection check failed: ${message}`);
|
|
113
|
+
await close();
|
|
114
|
+
process.exitCode = 1;
|
|
115
|
+
process.removeListener("SIGTERM", shutdown);
|
|
116
|
+
process.removeListener("SIGINT", shutdown);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
log.info(`Starting sync for ${options.site}...`);
|
|
64
120
|
// Find or create the gscSite document in Sanity
|
|
65
121
|
t = timer.start();
|
|
66
122
|
let siteDoc = await sanity.fetch(`*[_type == "gscSite" && siteUrl == $siteUrl][0]{
|
|
@@ -70,7 +126,7 @@ export const syncCommand = new Command("sync")
|
|
|
70
126
|
slugField
|
|
71
127
|
}`, { siteUrl: options.site });
|
|
72
128
|
if (!siteDoc) {
|
|
73
|
-
|
|
129
|
+
log.info(`Creating gscSite document for ${options.site}...`);
|
|
74
130
|
siteDoc = await sanity.create({
|
|
75
131
|
_type: "gscSite",
|
|
76
132
|
siteUrl: options.site,
|
|
@@ -85,15 +141,15 @@ export const syncCommand = new Command("sync")
|
|
|
85
141
|
const contentTypes = siteDoc.contentTypes ?? ["post", "page"];
|
|
86
142
|
const slugField = siteDoc.slugField ?? "slug";
|
|
87
143
|
const pathPrefix = siteDoc.pathPrefix ?? undefined;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
144
|
+
log.info(`Configuration:`);
|
|
145
|
+
log.info(` Content types: ${contentTypes.join(", ")}`);
|
|
146
|
+
log.info(` Slug field: ${slugField}`);
|
|
147
|
+
log.info(` Path prefix: ${pathPrefix ?? "(none)"}`);
|
|
92
148
|
const syncEngine = new SyncEngine({ gsc, db, sanity });
|
|
93
149
|
const matcher = new URLMatcher(sanity, {
|
|
94
150
|
contentTypes,
|
|
95
151
|
slugField,
|
|
96
|
-
baseUrl:
|
|
152
|
+
baseUrl: siteUrl,
|
|
97
153
|
pathPrefix,
|
|
98
154
|
});
|
|
99
155
|
try {
|
|
@@ -104,16 +160,16 @@ export const syncCommand = new Command("sync")
|
|
|
104
160
|
endDate: daysAgo(3),
|
|
105
161
|
});
|
|
106
162
|
timer.end("GSC data sync", t);
|
|
107
|
-
|
|
163
|
+
log.info(`Processed ${rowsProcessed} rows for ${pages.length} pages`);
|
|
108
164
|
t = timer.start();
|
|
109
165
|
const matches = await matcher.matchUrls(pages);
|
|
110
166
|
timer.end("URL matching", t);
|
|
111
167
|
const matched = matches.filter((m) => !!m.sanityId);
|
|
112
168
|
const unmatched = matches.filter((m) => !m.sanityId);
|
|
113
|
-
|
|
169
|
+
log.info(`Matched ${matched.length}/${pages.length} URLs to Sanity documents`);
|
|
114
170
|
// Store diagnostics for unmatched URLs
|
|
115
171
|
if (unmatched.length > 0) {
|
|
116
|
-
|
|
172
|
+
log.info(`${unmatched.length} unmatched URLs`);
|
|
117
173
|
// Store diagnostics in database
|
|
118
174
|
t = timer.start();
|
|
119
175
|
for (const u of unmatched) {
|
|
@@ -162,7 +218,7 @@ export const syncCommand = new Command("sync")
|
|
|
162
218
|
timer.end("Store unmatched diagnostics", t);
|
|
163
219
|
// Show detailed diagnostics if --diagnose flag is set
|
|
164
220
|
if (options.diagnose) {
|
|
165
|
-
|
|
221
|
+
log.info(`\nUnmatched URL Diagnostics:\n`);
|
|
166
222
|
// Group by reason
|
|
167
223
|
const byReason = new Map();
|
|
168
224
|
for (const u of unmatched) {
|
|
@@ -171,32 +227,31 @@ export const syncCommand = new Command("sync")
|
|
|
171
227
|
byReason.set(u.unmatchReason, existing);
|
|
172
228
|
}
|
|
173
229
|
for (const [reason, urls] of byReason) {
|
|
174
|
-
|
|
230
|
+
log.info(` ${getReasonEmoji(reason)} ${getReasonDescription(reason)} (${urls.length}):`);
|
|
175
231
|
const toShow = urls.slice(0, 5);
|
|
176
232
|
for (const u of toShow) {
|
|
177
|
-
|
|
233
|
+
log.info(` ${u.gscUrl}`);
|
|
178
234
|
if (u.extractedSlug) {
|
|
179
|
-
|
|
235
|
+
log.info(` Extracted slug: "${u.extractedSlug}"`);
|
|
180
236
|
}
|
|
181
237
|
if (u.diagnostics?.similarSlugs?.length) {
|
|
182
|
-
|
|
238
|
+
log.info(` Similar slugs in Sanity:`);
|
|
183
239
|
for (const similar of u.diagnostics.similarSlugs) {
|
|
184
|
-
|
|
240
|
+
log.info(` - ${similar}`);
|
|
185
241
|
}
|
|
186
242
|
}
|
|
187
243
|
}
|
|
188
244
|
if (urls.length > 5) {
|
|
189
|
-
|
|
245
|
+
log.info(` ... and ${urls.length - 5} more`);
|
|
190
246
|
}
|
|
191
|
-
console.log();
|
|
192
247
|
}
|
|
193
248
|
}
|
|
194
249
|
else if (unmatched.length <= 10) {
|
|
195
|
-
unmatched.forEach((u) =>
|
|
196
|
-
|
|
250
|
+
unmatched.forEach((u) => log.info(` - ${u.gscUrl}`));
|
|
251
|
+
log.info(`\n Run with --diagnose for detailed diagnostics`);
|
|
197
252
|
}
|
|
198
253
|
else {
|
|
199
|
-
|
|
254
|
+
log.info(` Run with --diagnose to see detailed diagnostics`);
|
|
200
255
|
}
|
|
201
256
|
}
|
|
202
257
|
// Handle --diagnose-url for a specific URL
|
|
@@ -204,25 +259,25 @@ export const syncCommand = new Command("sync")
|
|
|
204
259
|
const targetUrl = options.diagnoseUrl;
|
|
205
260
|
const allUrls = [targetUrl];
|
|
206
261
|
const [result] = await matcher.matchUrls(allUrls);
|
|
207
|
-
|
|
262
|
+
log.info(`\nDiagnostics for: ${targetUrl}\n`);
|
|
208
263
|
if (result) {
|
|
209
|
-
|
|
210
|
-
|
|
264
|
+
log.info(` Matched: ${result.sanityId ? "Yes" : "No"}`);
|
|
265
|
+
log.info(` Reason: ${getReasonDescription(result.unmatchReason)}`);
|
|
211
266
|
if (result.extractedSlug) {
|
|
212
|
-
|
|
267
|
+
log.info(` Extracted slug: "${result.extractedSlug}"`);
|
|
213
268
|
}
|
|
214
269
|
if (result.matchedSlug) {
|
|
215
|
-
|
|
270
|
+
log.info(` Matched to Sanity slug: "${result.matchedSlug}"`);
|
|
216
271
|
}
|
|
217
272
|
if (result.diagnostics) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
273
|
+
log.info(` Normalized URL: ${result.diagnostics.normalizedUrl}`);
|
|
274
|
+
log.info(` Path after prefix: ${result.diagnostics.pathAfterPrefix}`);
|
|
275
|
+
log.info(` Configured prefix: ${result.diagnostics.configuredPrefix ?? "(none)"}`);
|
|
276
|
+
log.info(` Available Sanity slugs: ${result.diagnostics.availableSlugsCount}`);
|
|
222
277
|
if (result.diagnostics.similarSlugs?.length) {
|
|
223
|
-
|
|
278
|
+
log.info(` Similar slugs in Sanity:`);
|
|
224
279
|
for (const similar of result.diagnostics.similarSlugs) {
|
|
225
|
-
|
|
280
|
+
log.info(` - ${similar}`);
|
|
226
281
|
}
|
|
227
282
|
}
|
|
228
283
|
}
|
|
@@ -230,12 +285,12 @@ export const syncCommand = new Command("sync")
|
|
|
230
285
|
}
|
|
231
286
|
// Check index status if requested
|
|
232
287
|
if (options.checkIndex && matched.length > 0) {
|
|
233
|
-
|
|
288
|
+
log.info(`\nChecking index status for ${matched.length} pages...`);
|
|
234
289
|
t = timer.start();
|
|
235
290
|
const matchedUrls = matched.map((m) => m.gscUrl);
|
|
236
291
|
const indexResult = await syncEngine.syncIndexStatus(options.site, matchedUrls);
|
|
237
292
|
timer.end("Index status check", t);
|
|
238
|
-
|
|
293
|
+
log.info(` Indexed: ${indexResult.indexed}, Not indexed: ${indexResult.notIndexed}, Skipped: ${indexResult.skipped}`);
|
|
239
294
|
}
|
|
240
295
|
if (!options.skipTasks) {
|
|
241
296
|
t = timer.start();
|
|
@@ -243,16 +298,16 @@ export const syncCommand = new Command("sync")
|
|
|
243
298
|
const detector = new DecayDetector(db);
|
|
244
299
|
const signals = await detector.detectDecay(options.site, publishedDates, {
|
|
245
300
|
enabled: true,
|
|
246
|
-
days:
|
|
301
|
+
days: quietPeriodDays,
|
|
247
302
|
});
|
|
248
303
|
timer.end("Decay detection", t);
|
|
249
|
-
|
|
304
|
+
log.info(`Detected ${signals.length} decay signals`);
|
|
250
305
|
if (options.dryRun) {
|
|
251
|
-
|
|
306
|
+
log.info("\nWould create the following tasks:");
|
|
252
307
|
signals.forEach((s) => {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
308
|
+
log.info(` [${s.severity.toUpperCase()}] ${s.page}`);
|
|
309
|
+
log.info(` Reason: ${s.reason}`);
|
|
310
|
+
log.info(` Position: ${s.metrics.positionBefore} -> ${s.metrics.positionNow}`);
|
|
256
311
|
});
|
|
257
312
|
}
|
|
258
313
|
else {
|
|
@@ -260,24 +315,27 @@ export const syncCommand = new Command("sync")
|
|
|
260
315
|
const taskGenerator = new TaskGenerator(sanity);
|
|
261
316
|
const created = await taskGenerator.createTasks(siteId, signals, matches);
|
|
262
317
|
timer.end("Task generation", t);
|
|
263
|
-
|
|
318
|
+
log.info(`Created ${created} new refresh tasks`);
|
|
264
319
|
}
|
|
265
320
|
}
|
|
266
321
|
if (!options.dryRun) {
|
|
267
322
|
t = timer.start();
|
|
268
323
|
await syncEngine.writeSnapshots(siteId, matched);
|
|
269
324
|
timer.end("Write Sanity snapshots", t);
|
|
270
|
-
|
|
325
|
+
log.info(`Updated Sanity snapshots`);
|
|
271
326
|
}
|
|
272
327
|
timer.end("Total sync", syncStartTime);
|
|
273
|
-
|
|
328
|
+
log.info(`\nSync complete!`);
|
|
274
329
|
}
|
|
275
330
|
catch (error) {
|
|
276
|
-
|
|
277
|
-
|
|
331
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
332
|
+
log.error(`Sync failed: ${message}`);
|
|
333
|
+
process.exitCode = 1;
|
|
278
334
|
}
|
|
279
335
|
finally {
|
|
280
|
-
await
|
|
336
|
+
await close();
|
|
337
|
+
process.removeListener("SIGTERM", shutdown);
|
|
338
|
+
process.removeListener("SIGINT", shutdown);
|
|
281
339
|
}
|
|
282
340
|
});
|
|
283
341
|
async function getPublishedDates(sanity, matches) {
|
package/dist/index.js
CHANGED
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { config } from "dotenv";
|
|
3
|
-
import { resolve } from "path";
|
|
4
|
-
import { fileURLToPath } from "url";
|
|
5
|
-
const __dirname = fileURLToPath(new URL(".", import.meta.url));
|
|
6
|
-
// Load .env from monorepo root
|
|
7
|
-
config({ path: resolve(__dirname, "../../../.env") });
|
|
8
2
|
import { program } from "commander";
|
|
9
3
|
import { syncCommand } from "./commands/sync.js";
|
|
10
4
|
import { listSitesCommand } from "./commands/list-sites.js";
|
package/dist/logger.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,GAAG;cACJ,MAAM;cAGN,MAAM;eAGL,MAAM;eAGN,MAAM,WAAW,OAAO;CAKpC,CAAC"}
|
package/dist/logger.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
function format(level, msg) {
|
|
2
|
+
return `[${new Date().toISOString()}] [${level}] ${msg}`;
|
|
3
|
+
}
|
|
4
|
+
export const log = {
|
|
5
|
+
info(msg) {
|
|
6
|
+
console.log(format("INFO", msg));
|
|
7
|
+
},
|
|
8
|
+
warn(msg) {
|
|
9
|
+
console.warn(format("WARN", msg));
|
|
10
|
+
},
|
|
11
|
+
error(msg) {
|
|
12
|
+
console.error(format("ERROR", msg));
|
|
13
|
+
},
|
|
14
|
+
debug(msg, enabled) {
|
|
15
|
+
if (enabled) {
|
|
16
|
+
console.log(format("DEBUG", msg));
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../src/migrate.ts"],"names":[],"mappings":"AAKA,wBAAsB,kBAAkB,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,iBAO7E"}
|
package/dist/migrate.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { runMigrations } from "@pagebridge/db";
|
|
4
|
+
import { log } from "./logger.js";
|
|
5
|
+
export async function migrateIfRequested(shouldMigrate, dbUrl) {
|
|
6
|
+
if (!shouldMigrate)
|
|
7
|
+
return;
|
|
8
|
+
log.info("Running database migrations...");
|
|
9
|
+
const pkgPath = import.meta.resolve("@pagebridge/db");
|
|
10
|
+
const migrationsFolder = resolve(fileURLToPath(pkgPath), "../../drizzle");
|
|
11
|
+
await runMigrations(dbUrl, migrationsFolder);
|
|
12
|
+
log.info("Migrations complete.");
|
|
13
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolves a config value from a CLI option first, then env var fallback.
|
|
3
|
+
*/
|
|
4
|
+
export declare function resolve(optionValue: string | undefined, envVarName: string): string | undefined;
|
|
5
|
+
interface ConfigEntry {
|
|
6
|
+
name: string;
|
|
7
|
+
flag: string;
|
|
8
|
+
envVar: string;
|
|
9
|
+
value: string | undefined;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Validates that all required config entries have values.
|
|
13
|
+
* If any are missing, prints a clear error listing every missing entry and exits.
|
|
14
|
+
*/
|
|
15
|
+
export declare function requireConfig(entries: ConfigEntry[]): void;
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=resolve-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-config.d.ts","sourceRoot":"","sources":["../src/resolve-config.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,OAAO,CACrB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,UAAU,EAAE,MAAM,GACjB,MAAM,GAAG,SAAS,CAEpB;AAED,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI,CAU1D"}
|