@ainyc/canonry 1.21.1 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/index-sl--69xx.js +246 -0
- package/assets/index.html +1 -1
- package/dist/{chunk-2JBQ7NMO.js → chunk-SN5AMQJE.js} +130 -43
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +6 -6
- package/assets/assets/index-BFA6dYNt.js +0 -246
package/assets/index.html
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
<link rel="icon" type="image/png" sizes="32x32" href="./favicon-32.png" />
|
|
13
13
|
<link rel="apple-touch-icon" href="./apple-touch-icon.png" />
|
|
14
14
|
<title>Canonry</title>
|
|
15
|
-
<script type="module" crossorigin src="./assets/index-
|
|
15
|
+
<script type="module" crossorigin src="./assets/index-sl--69xx.js"></script>
|
|
16
16
|
<link rel="stylesheet" crossorigin href="./assets/index-7DjD4Oje.css">
|
|
17
17
|
</head>
|
|
18
18
|
<body>
|
|
@@ -5221,6 +5221,11 @@ async function refreshAccessToken(clientId, clientSecret, currentRefreshToken) {
|
|
|
5221
5221
|
}
|
|
5222
5222
|
|
|
5223
5223
|
// ../integration-google/src/gsc-client.ts
|
|
5224
|
+
function gscClientLog(level, action, ctx) {
|
|
5225
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GscClient", action, ...ctx };
|
|
5226
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
5227
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
5228
|
+
}
|
|
5224
5229
|
async function gscFetch(accessToken, url, opts) {
|
|
5225
5230
|
const method = opts?.method ?? "GET";
|
|
5226
5231
|
const headers = {
|
|
@@ -5233,13 +5238,18 @@ async function gscFetch(accessToken, url, opts) {
|
|
|
5233
5238
|
body: opts?.body != null ? JSON.stringify(opts.body) : void 0
|
|
5234
5239
|
});
|
|
5235
5240
|
if (res.status === 401) {
|
|
5241
|
+
const body = await res.text().catch(() => "");
|
|
5242
|
+
gscClientLog("error", "http.auth-expired", { url, method, httpStatus: 401, responseBody: body });
|
|
5236
5243
|
throw new GoogleApiError("Access token expired or revoked", 401);
|
|
5237
5244
|
}
|
|
5238
5245
|
if (res.status === 429) {
|
|
5246
|
+
const body = await res.text().catch(() => "");
|
|
5247
|
+
gscClientLog("error", "http.rate-limited", { url, method, httpStatus: 429, responseBody: body });
|
|
5239
5248
|
throw new GoogleApiError("Google API rate limit exceeded", 429);
|
|
5240
5249
|
}
|
|
5241
5250
|
if (!res.ok) {
|
|
5242
5251
|
const body = await res.text();
|
|
5252
|
+
gscClientLog("error", "http.error", { url, method, httpStatus: res.status, responseBody: body });
|
|
5243
5253
|
throw new GoogleApiError(`GSC API error (${res.status}): ${body}`, res.status);
|
|
5244
5254
|
}
|
|
5245
5255
|
return await res.json();
|
|
@@ -6045,6 +6055,11 @@ var BingApiError = class extends Error {
|
|
|
6045
6055
|
};
|
|
6046
6056
|
|
|
6047
6057
|
// ../integration-bing/src/bing-client.ts
|
|
6058
|
+
function bingClientLog(level, action, ctx) {
|
|
6059
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "BingClient", action, ...ctx };
|
|
6060
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6061
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6062
|
+
}
|
|
6048
6063
|
async function bingFetch(apiKey, endpoint, opts) {
|
|
6049
6064
|
const method = opts?.method ?? "GET";
|
|
6050
6065
|
const separator = endpoint.includes("?") ? "&" : "?";
|
|
@@ -6058,13 +6073,18 @@ async function bingFetch(apiKey, endpoint, opts) {
|
|
|
6058
6073
|
body: opts?.body != null ? JSON.stringify(opts.body) : void 0
|
|
6059
6074
|
});
|
|
6060
6075
|
if (res.status === 401 || res.status === 403) {
|
|
6076
|
+
const body = await res.text().catch(() => "");
|
|
6077
|
+
bingClientLog("error", "http.auth-failed", { endpoint, method, httpStatus: res.status, responseBody: body });
|
|
6061
6078
|
throw new BingApiError("Bing API key is invalid or unauthorized", res.status);
|
|
6062
6079
|
}
|
|
6063
6080
|
if (res.status === 429) {
|
|
6081
|
+
const body = await res.text().catch(() => "");
|
|
6082
|
+
bingClientLog("error", "http.rate-limited", { endpoint, method, httpStatus: 429, responseBody: body });
|
|
6064
6083
|
throw new BingApiError("Bing API rate limit exceeded", 429);
|
|
6065
6084
|
}
|
|
6066
6085
|
if (!res.ok) {
|
|
6067
6086
|
const body = await res.text();
|
|
6087
|
+
bingClientLog("error", "http.error", { endpoint, method, httpStatus: res.status, responseBody: body });
|
|
6068
6088
|
throw new BingApiError(`Bing API error (${res.status}): ${body}`, res.status);
|
|
6069
6089
|
}
|
|
6070
6090
|
const text2 = await res.text();
|
|
@@ -6112,6 +6132,11 @@ async function getKeywordStats(apiKey, siteUrl) {
|
|
|
6112
6132
|
}
|
|
6113
6133
|
|
|
6114
6134
|
// ../api-routes/src/bing.ts
|
|
6135
|
+
function bingLog(level, action, ctx) {
|
|
6136
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "BingRoutes", action, ...ctx };
|
|
6137
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6138
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6139
|
+
}
|
|
6115
6140
|
async function bingRoutes(app, opts) {
|
|
6116
6141
|
function requireConnectionStore(reply) {
|
|
6117
6142
|
if (opts.bingConnectionStore) return opts.bingConnectionStore;
|
|
@@ -6140,8 +6165,10 @@ async function bingRoutes(app, opts) {
|
|
|
6140
6165
|
let sites;
|
|
6141
6166
|
try {
|
|
6142
6167
|
sites = await getSites(apiKey);
|
|
6168
|
+
bingLog("info", "connect.verify-key", { domain: project.canonicalDomain, siteCount: sites.length });
|
|
6143
6169
|
} catch (e) {
|
|
6144
6170
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6171
|
+
bingLog("error", "connect.verify-key-failed", { domain: project.canonicalDomain, error: msg });
|
|
6145
6172
|
const err = validationError(`Failed to verify Bing API key: ${msg}`);
|
|
6146
6173
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6147
6174
|
}
|
|
@@ -6307,7 +6334,15 @@ async function bingRoutes(app, opts) {
|
|
|
6307
6334
|
const err = validationError("url is required");
|
|
6308
6335
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6309
6336
|
}
|
|
6310
|
-
|
|
6337
|
+
let result;
|
|
6338
|
+
try {
|
|
6339
|
+
result = await getUrlInfo(conn.apiKey, conn.siteUrl, url);
|
|
6340
|
+
bingLog("info", "inspect-url.result", { domain: project.canonicalDomain, url, httpCode: result.HttpCode ?? null, inIndex: result.InIndex ?? null, lastCrawledDate: result.LastCrawledDate ?? null });
|
|
6341
|
+
} catch (e) {
|
|
6342
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
6343
|
+
bingLog("error", "inspect-url.failed", { domain: project.canonicalDomain, url, error: msg });
|
|
6344
|
+
throw e;
|
|
6345
|
+
}
|
|
6311
6346
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6312
6347
|
const id = crypto14.randomUUID();
|
|
6313
6348
|
app.db.insert(bingUrlInspections).values({
|
|
@@ -6371,6 +6406,7 @@ async function bingRoutes(app, opts) {
|
|
|
6371
6406
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6372
6407
|
}
|
|
6373
6408
|
const results = [];
|
|
6409
|
+
bingLog("info", "index-submit.start", { domain: project.canonicalDomain, siteUrl: conn.siteUrl, urlCount: urlsToSubmit.length, allUnindexed: !!request.body?.allUnindexed });
|
|
6374
6410
|
if (urlsToSubmit.length > 1) {
|
|
6375
6411
|
for (let i = 0; i < urlsToSubmit.length; i += BING_SUBMIT_URL_BATCH_LIMIT) {
|
|
6376
6412
|
const batch = urlsToSubmit.slice(i, i + BING_SUBMIT_URL_BATCH_LIMIT);
|
|
@@ -6380,12 +6416,14 @@ async function bingRoutes(app, opts) {
|
|
|
6380
6416
|
for (const url of batch) {
|
|
6381
6417
|
results.push({ url, status: "success", submittedAt: now });
|
|
6382
6418
|
}
|
|
6419
|
+
bingLog("info", "index-submit.batch-ok", { domain: project.canonicalDomain, batchSize: batch.length, urls: batch });
|
|
6383
6420
|
} catch (e) {
|
|
6384
6421
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6385
6422
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6386
6423
|
for (const url of batch) {
|
|
6387
6424
|
results.push({ url, status: "error", submittedAt: now, error: msg });
|
|
6388
6425
|
}
|
|
6426
|
+
bingLog("error", "index-submit.batch-failed", { domain: project.canonicalDomain, batchSize: batch.length, urls: batch, error: msg });
|
|
6389
6427
|
}
|
|
6390
6428
|
}
|
|
6391
6429
|
} else {
|
|
@@ -6393,13 +6431,16 @@ async function bingRoutes(app, opts) {
|
|
|
6393
6431
|
try {
|
|
6394
6432
|
await submitUrl(conn.apiKey, conn.siteUrl, url);
|
|
6395
6433
|
results.push({ url, status: "success", submittedAt: (/* @__PURE__ */ new Date()).toISOString() });
|
|
6434
|
+
bingLog("info", "index-submit.ok", { domain: project.canonicalDomain, url });
|
|
6396
6435
|
} catch (e) {
|
|
6397
6436
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6398
6437
|
results.push({ url, status: "error", submittedAt: (/* @__PURE__ */ new Date()).toISOString(), error: msg });
|
|
6438
|
+
bingLog("error", "index-submit.failed", { domain: project.canonicalDomain, url, error: msg });
|
|
6399
6439
|
}
|
|
6400
6440
|
}
|
|
6401
6441
|
const succeeded = results.filter((r) => r.status === "success").length;
|
|
6402
6442
|
const failed = results.filter((r) => r.status === "error").length;
|
|
6443
|
+
bingLog("info", "index-submit.complete", { domain: project.canonicalDomain, total: results.length, succeeded, failed });
|
|
6403
6444
|
return {
|
|
6404
6445
|
summary: { total: results.length, succeeded, failed },
|
|
6405
6446
|
results
|
|
@@ -8523,6 +8564,46 @@ import fs4 from "fs";
|
|
|
8523
8564
|
import path5 from "path";
|
|
8524
8565
|
import os4 from "os";
|
|
8525
8566
|
import { and as and6, eq as eq16, inArray as inArray3 } from "drizzle-orm";
|
|
8567
|
+
|
|
8568
|
+
// src/logger.ts
|
|
8569
|
+
var IS_TTY = process.stdout.isTTY === true;
|
|
8570
|
+
function formatTTY(entry) {
|
|
8571
|
+
const { ts, level, module, action, msg, ...ctx } = entry;
|
|
8572
|
+
const time = ts.slice(11, 19);
|
|
8573
|
+
const levelTag = level === "error" ? "\x1B[31mERR\x1B[0m" : level === "warn" ? "\x1B[33mWRN\x1B[0m" : "\x1B[36mINF\x1B[0m";
|
|
8574
|
+
const ctxParts = Object.entries(ctx).filter(([, v]) => v !== void 0 && v !== null).map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`).join(" ");
|
|
8575
|
+
const msgPart = msg ? ` ${msg}` : "";
|
|
8576
|
+
const ctxPart = ctxParts ? ` ${ctxParts}` : "";
|
|
8577
|
+
return `${time} ${levelTag} [${module}] ${action}${msgPart}${ctxPart}`;
|
|
8578
|
+
}
|
|
8579
|
+
function emit(entry) {
|
|
8580
|
+
const stream = entry.level === "error" ? process.stderr : process.stdout;
|
|
8581
|
+
if (IS_TTY) {
|
|
8582
|
+
stream.write(formatTTY(entry) + "\n");
|
|
8583
|
+
} else {
|
|
8584
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
8585
|
+
}
|
|
8586
|
+
}
|
|
8587
|
+
function createLogger(module) {
|
|
8588
|
+
function log7(level, action, ctx) {
|
|
8589
|
+
const entry = {
|
|
8590
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8591
|
+
level,
|
|
8592
|
+
module,
|
|
8593
|
+
action,
|
|
8594
|
+
...ctx
|
|
8595
|
+
};
|
|
8596
|
+
emit(entry);
|
|
8597
|
+
}
|
|
8598
|
+
return {
|
|
8599
|
+
info: (action, ctx) => log7("info", action, ctx),
|
|
8600
|
+
warn: (action, ctx) => log7("warn", action, ctx),
|
|
8601
|
+
error: (action, ctx) => log7("error", action, ctx)
|
|
8602
|
+
};
|
|
8603
|
+
}
|
|
8604
|
+
|
|
8605
|
+
// src/job-runner.ts
|
|
8606
|
+
var log = createLogger("JobRunner");
|
|
8526
8607
|
var RunCancelledError = class extends Error {
|
|
8527
8608
|
constructor(runId) {
|
|
8528
8609
|
super(`Run ${runId} was cancelled`);
|
|
@@ -8605,7 +8686,7 @@ var JobRunner = class {
|
|
|
8605
8686
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8606
8687
|
for (const run of stale) {
|
|
8607
8688
|
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq16(runs.id, run.id)).run();
|
|
8608
|
-
|
|
8689
|
+
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
8609
8690
|
}
|
|
8610
8691
|
}
|
|
8611
8692
|
async executeRun(runId, projectId, providerOverride, locationOverride) {
|
|
@@ -8654,7 +8735,7 @@ var JobRunner = class {
|
|
|
8654
8735
|
if (activeProviders.length === 0) {
|
|
8655
8736
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
8656
8737
|
}
|
|
8657
|
-
|
|
8738
|
+
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
8658
8739
|
projectKeywords = this.db.select().from(keywords).where(eq16(keywords.projectId, projectId)).all();
|
|
8659
8740
|
const projectCompetitors = this.db.select().from(competitors).where(eq16(competitors.projectId, projectId)).all();
|
|
8660
8741
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
@@ -8716,7 +8797,7 @@ var JobRunner = class {
|
|
|
8716
8797
|
);
|
|
8717
8798
|
this.throwIfRunCancelled(runId);
|
|
8718
8799
|
const normalized = adapter.normalizeResult(raw);
|
|
8719
|
-
|
|
8800
|
+
log.info("query.result", { runId, provider: providerName, keyword: kw.keyword, citedDomains: normalized.citedDomains, groundingSources: normalized.groundingSources.map((s) => s.uri), matchDomains: allDomains });
|
|
8720
8801
|
const citationState = determineCitationState(normalized, allDomains);
|
|
8721
8802
|
const overlap = computeCompetitorOverlap(normalized, competitorDomains);
|
|
8722
8803
|
let screenshotRelPath = null;
|
|
@@ -8769,14 +8850,15 @@ var JobRunner = class {
|
|
|
8769
8850
|
}).run();
|
|
8770
8851
|
}
|
|
8771
8852
|
totalSnapshotsInserted++;
|
|
8772
|
-
|
|
8853
|
+
log.info("query.citation", { runId, provider: providerName, keyword: kw.keyword, citationState });
|
|
8773
8854
|
});
|
|
8774
8855
|
} catch (err) {
|
|
8775
8856
|
if (err instanceof RunCancelledError) {
|
|
8776
8857
|
throw err;
|
|
8777
8858
|
}
|
|
8778
8859
|
const msg = err instanceof Error ? err.message : String(err);
|
|
8779
|
-
|
|
8860
|
+
const stack = err instanceof Error ? err.stack : void 0;
|
|
8861
|
+
log.error("query.failed", { runId, provider: providerName, keyword: kw.keyword, error: msg, stack });
|
|
8780
8862
|
if (!providerErrors.has(providerName)) {
|
|
8781
8863
|
providerErrors.set(providerName, msg);
|
|
8782
8864
|
}
|
|
@@ -8817,7 +8899,7 @@ var JobRunner = class {
|
|
|
8817
8899
|
this.incrementUsage(projectId, "runs", 1);
|
|
8818
8900
|
if (this.onRunCompleted) {
|
|
8819
8901
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
8820
|
-
|
|
8902
|
+
log.error("notification.callback-failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8821
8903
|
});
|
|
8822
8904
|
}
|
|
8823
8905
|
} catch (err) {
|
|
@@ -8911,7 +8993,7 @@ var JobRunner = class {
|
|
|
8911
8993
|
});
|
|
8912
8994
|
if (this.onRunCompleted) {
|
|
8913
8995
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
8914
|
-
|
|
8996
|
+
log.error("notification.callback-failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8915
8997
|
});
|
|
8916
8998
|
}
|
|
8917
8999
|
}
|
|
@@ -8984,6 +9066,7 @@ function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
|
8984
9066
|
// src/gsc-sync.ts
|
|
8985
9067
|
import crypto16 from "crypto";
|
|
8986
9068
|
import { eq as eq17, and as and7, sql as sql3 } from "drizzle-orm";
|
|
9069
|
+
var log2 = createLogger("GscSync");
|
|
8987
9070
|
function formatDate(d) {
|
|
8988
9071
|
return d.toISOString().split("T")[0];
|
|
8989
9072
|
}
|
|
@@ -9027,12 +9110,12 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9027
9110
|
const endDate = formatDate(daysAgo(lagOffset));
|
|
9028
9111
|
const days = opts.full ? 480 : opts.days ?? 30;
|
|
9029
9112
|
const startDate = formatDate(daysAgo(days + lagOffset));
|
|
9030
|
-
|
|
9113
|
+
log2.info("fetch.start", { runId, projectId, propertyId: conn.propertyId, startDate, endDate });
|
|
9031
9114
|
const rows = await fetchSearchAnalytics(accessToken, conn.propertyId, {
|
|
9032
9115
|
startDate,
|
|
9033
9116
|
endDate
|
|
9034
9117
|
});
|
|
9035
|
-
|
|
9118
|
+
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
9036
9119
|
db.delete(gscSearchData).where(
|
|
9037
9120
|
and7(
|
|
9038
9121
|
eq17(gscSearchData.projectId, projectId),
|
|
@@ -9071,7 +9154,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9071
9154
|
}
|
|
9072
9155
|
}
|
|
9073
9156
|
const topPages = [...pageClicks.entries()].sort((a, b) => b[1] - a[1]).slice(0, 50).map(([page]) => page);
|
|
9074
|
-
|
|
9157
|
+
log2.info("inspect.start", { runId, projectId, urlCount: topPages.length });
|
|
9075
9158
|
for (const pageUrl of topPages) {
|
|
9076
9159
|
try {
|
|
9077
9160
|
const result = await inspectUrl(accessToken, pageUrl, conn.propertyId);
|
|
@@ -9099,7 +9182,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9099
9182
|
createdAt: inspectedAt
|
|
9100
9183
|
}).run();
|
|
9101
9184
|
} catch (err) {
|
|
9102
|
-
|
|
9185
|
+
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
9103
9186
|
}
|
|
9104
9187
|
}
|
|
9105
9188
|
const allInspections = db.select().from(gscUrlInspections).where(eq17(gscUrlInspections.projectId, projectId)).all();
|
|
@@ -9135,11 +9218,11 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9135
9218
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9136
9219
|
}).run();
|
|
9137
9220
|
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq17(runs.id, runId)).run();
|
|
9138
|
-
|
|
9221
|
+
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9139
9222
|
} catch (err) {
|
|
9140
9223
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9141
9224
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq17(runs.id, runId)).run();
|
|
9142
|
-
|
|
9225
|
+
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
9143
9226
|
throw err;
|
|
9144
9227
|
}
|
|
9145
9228
|
}
|
|
@@ -9212,6 +9295,7 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
9212
9295
|
}
|
|
9213
9296
|
|
|
9214
9297
|
// src/gsc-inspect-sitemap.ts
|
|
9298
|
+
var log3 = createLogger("InspectSitemap");
|
|
9215
9299
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
9216
9300
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9217
9301
|
db.update(runs).set({ status: "running", startedAt: now }).where(eq18(runs.id, runId)).run();
|
|
@@ -9244,9 +9328,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9244
9328
|
saveConfig(opts.config);
|
|
9245
9329
|
}
|
|
9246
9330
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
9247
|
-
|
|
9331
|
+
log3.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
9248
9332
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
9249
|
-
|
|
9333
|
+
log3.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
9250
9334
|
if (urls.length === 0) {
|
|
9251
9335
|
throw new Error("No URLs found in sitemap");
|
|
9252
9336
|
}
|
|
@@ -9279,10 +9363,10 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9279
9363
|
createdAt: inspectedAt
|
|
9280
9364
|
}).run();
|
|
9281
9365
|
inspected++;
|
|
9282
|
-
|
|
9366
|
+
log3.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${urls.length}` });
|
|
9283
9367
|
} catch (err) {
|
|
9284
9368
|
errors++;
|
|
9285
|
-
|
|
9369
|
+
log3.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
9286
9370
|
}
|
|
9287
9371
|
if (inspected + errors < urls.length) {
|
|
9288
9372
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
@@ -9322,11 +9406,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9322
9406
|
}).run();
|
|
9323
9407
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
9324
9408
|
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9325
|
-
|
|
9409
|
+
log3.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9326
9410
|
} catch (err) {
|
|
9327
9411
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9328
9412
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9329
|
-
|
|
9413
|
+
log3.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
9330
9414
|
throw err;
|
|
9331
9415
|
}
|
|
9332
9416
|
}
|
|
@@ -9385,6 +9469,7 @@ var ProviderRegistry = class {
|
|
|
9385
9469
|
// src/scheduler.ts
|
|
9386
9470
|
import cron from "node-cron";
|
|
9387
9471
|
import { eq as eq19 } from "drizzle-orm";
|
|
9472
|
+
var log4 = createLogger("Scheduler");
|
|
9388
9473
|
var Scheduler = class {
|
|
9389
9474
|
db;
|
|
9390
9475
|
callbacks;
|
|
@@ -9400,11 +9485,11 @@ var Scheduler = class {
|
|
|
9400
9485
|
const missedRunAt = schedule.nextRunAt;
|
|
9401
9486
|
this.registerCronTask(schedule);
|
|
9402
9487
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
9403
|
-
|
|
9488
|
+
log4.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
9404
9489
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
9405
9490
|
}
|
|
9406
9491
|
}
|
|
9407
|
-
|
|
9492
|
+
log4.info("started", { scheduleCount: allSchedules.length });
|
|
9408
9493
|
}
|
|
9409
9494
|
/** Stop all cron tasks for graceful shutdown. */
|
|
9410
9495
|
stop() {
|
|
@@ -9436,12 +9521,12 @@ var Scheduler = class {
|
|
|
9436
9521
|
stopTask(projectId, task, verb) {
|
|
9437
9522
|
task.stop();
|
|
9438
9523
|
task.destroy();
|
|
9439
|
-
|
|
9524
|
+
log4.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
9440
9525
|
}
|
|
9441
9526
|
registerCronTask(schedule) {
|
|
9442
9527
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
9443
9528
|
if (!cron.validate(cronExpr)) {
|
|
9444
|
-
|
|
9529
|
+
log4.error("cron.invalid", { projectId, cronExpr });
|
|
9445
9530
|
return;
|
|
9446
9531
|
}
|
|
9447
9532
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -9455,13 +9540,13 @@ var Scheduler = class {
|
|
|
9455
9540
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9456
9541
|
}).where(eq19(schedules.id, scheduleId)).run();
|
|
9457
9542
|
const label = schedule.preset ?? cronExpr;
|
|
9458
|
-
|
|
9543
|
+
log4.info("cron.registered", { projectId, schedule: label, timezone });
|
|
9459
9544
|
}
|
|
9460
9545
|
triggerRun(scheduleId, projectId) {
|
|
9461
9546
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9462
9547
|
const currentSchedule = this.db.select().from(schedules).where(eq19(schedules.id, scheduleId)).get();
|
|
9463
9548
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
9464
|
-
|
|
9549
|
+
log4.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
9465
9550
|
this.remove(projectId);
|
|
9466
9551
|
return;
|
|
9467
9552
|
}
|
|
@@ -9469,7 +9554,7 @@ var Scheduler = class {
|
|
|
9469
9554
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
9470
9555
|
const project = this.db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
9471
9556
|
if (!project) {
|
|
9472
|
-
|
|
9557
|
+
log4.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
9473
9558
|
this.remove(projectId);
|
|
9474
9559
|
return;
|
|
9475
9560
|
}
|
|
@@ -9480,7 +9565,7 @@ var Scheduler = class {
|
|
|
9480
9565
|
trigger: "scheduled"
|
|
9481
9566
|
});
|
|
9482
9567
|
if (queueResult.conflict) {
|
|
9483
|
-
|
|
9568
|
+
log4.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
9484
9569
|
this.db.update(schedules).set({
|
|
9485
9570
|
nextRunAt,
|
|
9486
9571
|
updatedAt: now
|
|
@@ -9495,7 +9580,7 @@ var Scheduler = class {
|
|
|
9495
9580
|
}).where(eq19(schedules.id, currentSchedule.id)).run();
|
|
9496
9581
|
const scheduleProviders = JSON.parse(currentSchedule.providers);
|
|
9497
9582
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
9498
|
-
|
|
9583
|
+
log4.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
9499
9584
|
this.callbacks.onRunCreated(runId, projectId, providers);
|
|
9500
9585
|
}
|
|
9501
9586
|
};
|
|
@@ -9503,6 +9588,7 @@ var Scheduler = class {
|
|
|
9503
9588
|
// src/notifier.ts
|
|
9504
9589
|
import { eq as eq20, desc as desc6, and as and9, or as or2 } from "drizzle-orm";
|
|
9505
9590
|
import crypto18 from "crypto";
|
|
9591
|
+
var log5 = createLogger("Notifier");
|
|
9506
9592
|
var Notifier = class {
|
|
9507
9593
|
db;
|
|
9508
9594
|
serverUrl;
|
|
@@ -9512,26 +9598,26 @@ var Notifier = class {
|
|
|
9512
9598
|
}
|
|
9513
9599
|
/** Called after a run completes (success, partial, or failed). */
|
|
9514
9600
|
async onRunCompleted(runId, projectId) {
|
|
9515
|
-
|
|
9601
|
+
log5.info("run.completed", { runId, projectId });
|
|
9516
9602
|
const notifs = this.db.select().from(notifications).where(eq20(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
9517
9603
|
if (notifs.length === 0) {
|
|
9518
|
-
|
|
9604
|
+
log5.info("notifications.none-enabled", { projectId });
|
|
9519
9605
|
return;
|
|
9520
9606
|
}
|
|
9521
|
-
|
|
9607
|
+
log5.info("notifications.found", { projectId, count: notifs.length });
|
|
9522
9608
|
const run = this.db.select().from(runs).where(eq20(runs.id, runId)).get();
|
|
9523
9609
|
if (!run) {
|
|
9524
|
-
|
|
9610
|
+
log5.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
9525
9611
|
return;
|
|
9526
9612
|
}
|
|
9527
9613
|
const project = this.db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
9528
9614
|
if (!project) {
|
|
9529
|
-
|
|
9615
|
+
log5.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
9530
9616
|
return;
|
|
9531
9617
|
}
|
|
9532
9618
|
const transitions = this.computeTransitions(runId, projectId);
|
|
9533
9619
|
const events = [];
|
|
9534
|
-
|
|
9620
|
+
log5.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
9535
9621
|
if (run.status === "completed" || run.status === "partial") {
|
|
9536
9622
|
events.push("run.completed");
|
|
9537
9623
|
}
|
|
@@ -9546,7 +9632,7 @@ var Notifier = class {
|
|
|
9546
9632
|
const config = JSON.parse(notif.config);
|
|
9547
9633
|
const subscribedEvents = config.events;
|
|
9548
9634
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
9549
|
-
|
|
9635
|
+
log5.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
9550
9636
|
if (matchingEvents.length === 0) continue;
|
|
9551
9637
|
for (const event of matchingEvents) {
|
|
9552
9638
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -9606,23 +9692,23 @@ var Notifier = class {
|
|
|
9606
9692
|
async sendWebhook(url, payload, notificationId, projectId, webhookSecret) {
|
|
9607
9693
|
const targetCheck = await resolveWebhookTarget(url);
|
|
9608
9694
|
if (!targetCheck.ok) {
|
|
9609
|
-
|
|
9695
|
+
log5.error("webhook.ssrf-blocked", { url, reason: targetCheck.message });
|
|
9610
9696
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
9611
9697
|
return;
|
|
9612
9698
|
}
|
|
9613
|
-
|
|
9699
|
+
log5.info("webhook.send", { event: payload.event, url });
|
|
9614
9700
|
const maxRetries = 3;
|
|
9615
9701
|
const delays = [1e3, 4e3, 16e3];
|
|
9616
9702
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
9617
9703
|
try {
|
|
9618
9704
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
9619
9705
|
if (response.status >= 200 && response.status < 300) {
|
|
9620
|
-
|
|
9706
|
+
log5.info("webhook.delivered", { event: payload.event, url, httpStatus: response.status });
|
|
9621
9707
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
9622
9708
|
return;
|
|
9623
9709
|
}
|
|
9624
9710
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
9625
|
-
|
|
9711
|
+
log5.warn("webhook.attempt-failed", { event: payload.event, url, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
9626
9712
|
if (attempt === maxRetries - 1) {
|
|
9627
9713
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
9628
9714
|
}
|
|
@@ -9630,7 +9716,7 @@ var Notifier = class {
|
|
|
9630
9716
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
9631
9717
|
if (attempt === maxRetries - 1) {
|
|
9632
9718
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
9633
|
-
|
|
9719
|
+
log5.error("webhook.exhausted", { event: payload.event, url, maxRetries, error: errorDetail });
|
|
9634
9720
|
}
|
|
9635
9721
|
}
|
|
9636
9722
|
if (attempt < maxRetries - 1) {
|
|
@@ -9766,6 +9852,7 @@ function stripHtml(html) {
|
|
|
9766
9852
|
// src/server.ts
|
|
9767
9853
|
var _require2 = createRequire2(import.meta.url);
|
|
9768
9854
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
9855
|
+
var log6 = createLogger("Server");
|
|
9769
9856
|
var DEFAULT_QUOTA = {
|
|
9770
9857
|
maxConcurrency: 2,
|
|
9771
9858
|
maxRequestsPerMinute: 10,
|
|
@@ -9817,10 +9904,10 @@ async function createServer(opts) {
|
|
|
9817
9904
|
quota: opts.config.geminiQuota
|
|
9818
9905
|
};
|
|
9819
9906
|
}
|
|
9820
|
-
|
|
9907
|
+
log6.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
9821
9908
|
const p = providers[k];
|
|
9822
9909
|
return p?.apiKey || p?.baseUrl;
|
|
9823
|
-
}));
|
|
9910
|
+
}) });
|
|
9824
9911
|
for (const adapter of API_ADAPTERS) {
|
|
9825
9912
|
const entry = providers[adapter.name];
|
|
9826
9913
|
if (!entry) continue;
|
package/dist/cli.js
CHANGED
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ainyc/canonry",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.22.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The ultimate open-source AEO monitoring tool - track how answer engines cite your domain",
|
|
6
6
|
"license": "FSL-1.1-ALv2",
|
|
@@ -53,17 +53,17 @@
|
|
|
53
53
|
"tsup": "^8.5.1",
|
|
54
54
|
"tsx": "^4.19.0",
|
|
55
55
|
"@ainyc/canonry-api-routes": "0.0.0",
|
|
56
|
+
"@ainyc/canonry-contracts": "0.0.0",
|
|
56
57
|
"@ainyc/canonry-provider-claude": "0.0.0",
|
|
57
|
-
"@ainyc/canonry-db": "0.0.0",
|
|
58
|
-
"@ainyc/canonry-config": "0.0.0",
|
|
59
58
|
"@ainyc/canonry-provider-gemini": "0.0.0",
|
|
59
|
+
"@ainyc/canonry-config": "0.0.0",
|
|
60
|
+
"@ainyc/canonry-db": "0.0.0",
|
|
60
61
|
"@ainyc/canonry-provider-cdp": "0.0.0",
|
|
61
62
|
"@ainyc/canonry-integration-bing": "0.0.0",
|
|
62
63
|
"@ainyc/canonry-provider-local": "0.0.0",
|
|
63
|
-
"@ainyc/canonry-
|
|
64
|
-
"@ainyc/canonry-integration-google": "0.0.0",
|
|
64
|
+
"@ainyc/canonry-provider-openai": "0.0.0",
|
|
65
65
|
"@ainyc/canonry-provider-perplexity": "0.0.0",
|
|
66
|
-
"@ainyc/canonry-
|
|
66
|
+
"@ainyc/canonry-integration-google": "0.0.0"
|
|
67
67
|
},
|
|
68
68
|
"scripts": {
|
|
69
69
|
"build": "tsup && tsx build-web.ts",
|