@canonry/canonry 4.172.2 → 4.172.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +9 -7
- package/assets/assets/{AuditHistoryPanel-DnqySUw3.js → AuditHistoryPanel-CARZK_Wi.js} +1 -1
- package/assets/assets/{BacklinksPage-CbcHWD-G.js → BacklinksPage-BEA9vbC2.js} +1 -1
- package/assets/assets/{HistoryPage-BYFSyxuB.js → HistoryPage-DKcDQw1a.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BBnXQPIv.js → MeasurementPropertyPage-BRPI_icI.js} +1 -1
- package/assets/assets/ProjectPage-B4_EEP8k.js +9 -0
- package/assets/assets/{RunRow-lfyHktTf.js → RunRow-C8C7wrzp.js} +1 -1
- package/assets/assets/{RunsPage-ngnP0Y_D.js → RunsPage-BeotkZcg.js} +1 -1
- package/assets/assets/{SettingsPage-DEg74T20.js → SettingsPage-BukBukvW.js} +1 -1
- package/assets/assets/{SiteHealthSection-C6ycXwbt.js → SiteHealthSection-C7RuR9AA.js} +3 -3
- package/assets/assets/{TrafficPage-hrJ1IDqA.js → TrafficPage-D_zPUxGq.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-fu7nPl3E.js → TrafficSourceDetailPage-CetULBhn.js} +1 -1
- package/assets/assets/{extract-error-message-dcdO9njP.js → extract-error-message-CDRTdeQI.js} +1 -1
- package/assets/assets/{index-D1mXm81Z.css → index-CFzo8iAY.css} +1 -1
- package/assets/assets/{index-DpQ_-QC_.js → index-D156-4Hr.js} +19 -19
- package/assets/assets/{react-sigma_core.esm.min-DZGxGBSe.js → react-sigma_core.esm.min-CU8AjIPg.js} +1 -1
- package/assets/index.html +2 -2
- package/dist/{chunk-PXTSQ67Z.js → chunk-2CXNT2AC.js} +13083 -1260
- package/dist/{chunk-YXDCVM5U.js → chunk-BZDNTUCZ.js} +1 -1
- package/dist/{chunk-TI3SZAVI.js → chunk-N53TZMMR.js} +178 -168
- package/dist/{chunk-U5JTZM66.js → chunk-WFVK3XC7.js} +2 -2
- package/dist/{chunk-EGGQSFQL.js → chunk-WGN2VT7L.js} +49 -1
- package/dist/cli.js +40 -5
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-H7756PBD.js → intelligence-service-2E7F6I7I.js} +2 -2
- package/dist/mcp.js +3 -3
- package/package.json +12 -12
- package/assets/assets/ProjectPage-Cq5bHpmk.js +0 -9
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
loadConfig,
|
|
10
10
|
loadConfigRaw,
|
|
11
11
|
saveConfigPatch
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-BZDNTUCZ.js";
|
|
13
13
|
import {
|
|
14
14
|
CC_CACHE_DIR,
|
|
15
15
|
DUCKDB_SPEC,
|
|
@@ -145,7 +145,7 @@ import {
|
|
|
145
145
|
siteCrawlSnapshots,
|
|
146
146
|
toAlertView,
|
|
147
147
|
usageCounters
|
|
148
|
-
} from "./chunk-
|
|
148
|
+
} from "./chunk-2CXNT2AC.js";
|
|
149
149
|
import {
|
|
150
150
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
151
151
|
AGENT_PROVIDER_IDS,
|
|
@@ -282,7 +282,7 @@ import {
|
|
|
282
282
|
validationError,
|
|
283
283
|
winnabilityClassLabel,
|
|
284
284
|
withRetry
|
|
285
|
-
} from "./chunk-
|
|
285
|
+
} from "./chunk-WGN2VT7L.js";
|
|
286
286
|
|
|
287
287
|
// src/telemetry.ts
|
|
288
288
|
import crypto from "crypto";
|
|
@@ -2564,6 +2564,7 @@ function removeOrphanedGtmConnections(config, projectIds) {
|
|
|
2564
2564
|
}
|
|
2565
2565
|
|
|
2566
2566
|
// src/google-marketing-runtime.ts
|
|
2567
|
+
var log = createLogger("GoogleMarketing");
|
|
2567
2568
|
var TOKEN_REFRESH_SKEW_MS = 5 * 60 * 1e3;
|
|
2568
2569
|
var DEFAULT_LIST_LIMIT = 100;
|
|
2569
2570
|
var MAX_LIST_LIMIT = 1e3;
|
|
@@ -3006,6 +3007,15 @@ var DefaultGoogleMarketingRuntime = class {
|
|
|
3006
3007
|
}
|
|
3007
3008
|
}
|
|
3008
3009
|
const customers = [...customersById.values()].sort((left, right) => left.level - right.level || left.customerId.localeCompare(right.customerId));
|
|
3010
|
+
if (result.data.failures.length > 0) {
|
|
3011
|
+
log.error("google-ads.customer-details-failed", {
|
|
3012
|
+
project: normalizedProject.name,
|
|
3013
|
+
totalAccessible: result.data.totalAccessible,
|
|
3014
|
+
attempted: result.data.attempted,
|
|
3015
|
+
returned: customers.length,
|
|
3016
|
+
failures: result.data.failures
|
|
3017
|
+
});
|
|
3018
|
+
}
|
|
3009
3019
|
return googleAdsAccessibleCustomersResponseSchema.parse({
|
|
3010
3020
|
customers,
|
|
3011
3021
|
totalAccessible: customers.length,
|
|
@@ -6274,7 +6284,7 @@ async function captureCitedUrls(provider, sources, opts = {}) {
|
|
|
6274
6284
|
}
|
|
6275
6285
|
|
|
6276
6286
|
// src/job-runner.ts
|
|
6277
|
-
var
|
|
6287
|
+
var log2 = createLogger("JobRunner");
|
|
6278
6288
|
var RunCancelledError = class extends Error {
|
|
6279
6289
|
constructor(runId) {
|
|
6280
6290
|
super(`Run ${runId} was cancelled`);
|
|
@@ -6408,7 +6418,7 @@ var JobRunner = class {
|
|
|
6408
6418
|
return true;
|
|
6409
6419
|
});
|
|
6410
6420
|
if (!recovered) continue;
|
|
6411
|
-
|
|
6421
|
+
log2.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
6412
6422
|
}
|
|
6413
6423
|
}
|
|
6414
6424
|
async executeRun(runId, projectId, providerOverride, locationOverride) {
|
|
@@ -6478,7 +6488,7 @@ var JobRunner = class {
|
|
|
6478
6488
|
if (activeProviders.length === 0) {
|
|
6479
6489
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
6480
6490
|
}
|
|
6481
|
-
|
|
6491
|
+
log2.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
6482
6492
|
const projectCompetitors = this.db.select().from(competitors).where(eq2(competitors.projectId, projectId)).all();
|
|
6483
6493
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
6484
6494
|
const allDomains = effectiveDomains({
|
|
@@ -6563,7 +6573,7 @@ var JobRunner = class {
|
|
|
6563
6573
|
resolvedCount: 0,
|
|
6564
6574
|
captureVersion: CITED_URL_CAPTURE_VERSION
|
|
6565
6575
|
};
|
|
6566
|
-
|
|
6576
|
+
log2.warn("query.cited-url-capture-failed", {
|
|
6567
6577
|
runId,
|
|
6568
6578
|
provider: providerName,
|
|
6569
6579
|
query: q.query,
|
|
@@ -6571,7 +6581,7 @@ var JobRunner = class {
|
|
|
6571
6581
|
});
|
|
6572
6582
|
}
|
|
6573
6583
|
this.throwIfRunCancelled(runId);
|
|
6574
|
-
|
|
6584
|
+
log2.info("query.result", { runId, provider: providerName, query: q.query, citedDomains: normalized.citedDomains, groundingSources: normalized.groundingSources.map((s) => s.uri), matchDomains: allDomains });
|
|
6575
6585
|
const citationState = determineCitationState(normalized, allDomains);
|
|
6576
6586
|
const answerMentioned = determineAnswerMentioned(
|
|
6577
6587
|
normalized.answerText,
|
|
@@ -6666,7 +6676,7 @@ var JobRunner = class {
|
|
|
6666
6676
|
}).run();
|
|
6667
6677
|
}
|
|
6668
6678
|
totalSnapshotsInserted++;
|
|
6669
|
-
|
|
6679
|
+
log2.info("query.citation", { runId, provider: providerName, query: q.query, citationState, answerMentioned });
|
|
6670
6680
|
});
|
|
6671
6681
|
} catch (err) {
|
|
6672
6682
|
if (err instanceof RunCancelledError) {
|
|
@@ -6674,7 +6684,7 @@ var JobRunner = class {
|
|
|
6674
6684
|
}
|
|
6675
6685
|
const msg = describeError(err);
|
|
6676
6686
|
const stack = err instanceof Error ? err.stack : void 0;
|
|
6677
|
-
|
|
6687
|
+
log2.error("query.failed", { runId, provider: providerName, query: q.query, error: msg, stack });
|
|
6678
6688
|
if (!providerErrors.has(providerName)) {
|
|
6679
6689
|
providerErrors.set(providerName, msg);
|
|
6680
6690
|
}
|
|
@@ -6721,7 +6731,7 @@ var JobRunner = class {
|
|
|
6721
6731
|
resolvedCount: 0,
|
|
6722
6732
|
captureVersion: CITED_URL_CAPTURE_VERSION
|
|
6723
6733
|
};
|
|
6724
|
-
|
|
6734
|
+
log2.warn("query.cited-url-capture-failed", {
|
|
6725
6735
|
runId,
|
|
6726
6736
|
provider: providerName,
|
|
6727
6737
|
query: unit.queryText,
|
|
@@ -6798,7 +6808,7 @@ var JobRunner = class {
|
|
|
6798
6808
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6799
6809
|
}).run();
|
|
6800
6810
|
totalSnapshotsInserted++;
|
|
6801
|
-
|
|
6811
|
+
log2.info("query.citation", {
|
|
6802
6812
|
runId,
|
|
6803
6813
|
provider: providerName,
|
|
6804
6814
|
query: unit.queryText,
|
|
@@ -6814,7 +6824,7 @@ var JobRunner = class {
|
|
|
6814
6824
|
}
|
|
6815
6825
|
const msg = describeError(err);
|
|
6816
6826
|
const stack = err instanceof Error ? err.stack : void 0;
|
|
6817
|
-
|
|
6827
|
+
log2.error("query.failed", { runId, provider: providerName, query: unit.queryText, executionId: unit.executionId, error: msg, stack });
|
|
6818
6828
|
if (!providerErrors.has(providerName)) {
|
|
6819
6829
|
providerErrors.set(providerName, msg);
|
|
6820
6830
|
}
|
|
@@ -6824,7 +6834,7 @@ var JobRunner = class {
|
|
|
6824
6834
|
if (planExecution) {
|
|
6825
6835
|
const plan = planExecution;
|
|
6826
6836
|
const unitsFor = (provider) => plan.unitsByProvider.get(provider.adapter.name.trim().toLocaleLowerCase("en")) ?? [];
|
|
6827
|
-
|
|
6837
|
+
log2.info("run.plan-dispatch", {
|
|
6828
6838
|
runId,
|
|
6829
6839
|
expectedSlots: plan.manifest.expectedSlots.length,
|
|
6830
6840
|
executionNodes: plan.nodeCount,
|
|
@@ -6912,7 +6922,7 @@ var JobRunner = class {
|
|
|
6912
6922
|
this.incrementUsage(projectId, "runs", 1);
|
|
6913
6923
|
if (this.onRunCompleted) {
|
|
6914
6924
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
6915
|
-
|
|
6925
|
+
log2.error("notification.callback-failed", { runId, error: describeError(err) });
|
|
6916
6926
|
});
|
|
6917
6927
|
}
|
|
6918
6928
|
} catch (err) {
|
|
@@ -6970,7 +6980,7 @@ var JobRunner = class {
|
|
|
6970
6980
|
}
|
|
6971
6981
|
if (this.onRunCompleted) {
|
|
6972
6982
|
this.onRunCompleted(runId, projectId).catch((notifErr) => {
|
|
6973
|
-
|
|
6983
|
+
log2.error("notification.callback-failed", { runId, error: describeError(notifErr) });
|
|
6974
6984
|
});
|
|
6975
6985
|
}
|
|
6976
6986
|
}
|
|
@@ -7050,7 +7060,7 @@ var JobRunner = class {
|
|
|
7050
7060
|
);
|
|
7051
7061
|
if (this.onRunCompleted) {
|
|
7052
7062
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
7053
|
-
|
|
7063
|
+
log2.error("notification.callback-failed", { runId, error: describeError(err) });
|
|
7054
7064
|
});
|
|
7055
7065
|
}
|
|
7056
7066
|
}
|
|
@@ -7169,7 +7179,7 @@ function writeCoverageSnapshot(db, projectId, runId, opts = {}) {
|
|
|
7169
7179
|
}
|
|
7170
7180
|
|
|
7171
7181
|
// src/gsc-sync.ts
|
|
7172
|
-
var
|
|
7182
|
+
var log3 = createLogger("GscSync");
|
|
7173
7183
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
7174
7184
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7175
7185
|
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
@@ -7206,12 +7216,12 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
7206
7216
|
const endDate = formatIsoDateInTimeZone((/* @__PURE__ */ new Date()).toISOString(), GSC_REPORTING_TIME_ZONE);
|
|
7207
7217
|
const days = opts.full ? 480 : opts.days ?? 30;
|
|
7208
7218
|
const startDate = shiftIsoCalendarDate(endDate, -(days + lagOffset));
|
|
7209
|
-
|
|
7219
|
+
log3.info("fetch.start", { runId, projectId, propertyId: conn.propertyId, startDate, endDate });
|
|
7210
7220
|
const rows = await fetchSearchAnalytics(accessToken, conn.propertyId, {
|
|
7211
7221
|
startDate,
|
|
7212
7222
|
endDate
|
|
7213
7223
|
});
|
|
7214
|
-
|
|
7224
|
+
log3.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
7215
7225
|
db.delete(gscSearchData).where(
|
|
7216
7226
|
and4(
|
|
7217
7227
|
eq4(gscSearchData.projectId, projectId),
|
|
@@ -7283,7 +7293,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
7283
7293
|
}).run();
|
|
7284
7294
|
}
|
|
7285
7295
|
}
|
|
7286
|
-
|
|
7296
|
+
log3.info("daily-totals.complete", {
|
|
7287
7297
|
runId,
|
|
7288
7298
|
projectId,
|
|
7289
7299
|
rowCount: totalRows.length,
|
|
@@ -7319,14 +7329,14 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
7319
7329
|
createdAt: queryTotalsNow
|
|
7320
7330
|
}).run();
|
|
7321
7331
|
}
|
|
7322
|
-
|
|
7332
|
+
log3.info("query-totals.complete", { runId, projectId, rowCount: queryTotalRows.length });
|
|
7323
7333
|
const coverage = writeCoverageSnapshot(db, projectId, runId);
|
|
7324
7334
|
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
7325
|
-
|
|
7335
|
+
log3.info("sync.completed", { runId, projectId, searchDataRows: rows.length, indexed: coverage.indexed, notIndexed: coverage.notIndexed, unknown: coverage.unknown, verifiedByInspection: coverage.verifiedByInspection });
|
|
7326
7336
|
} catch (err) {
|
|
7327
7337
|
const errorMsg = describeError(err);
|
|
7328
7338
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
7329
|
-
|
|
7339
|
+
log3.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
7330
7340
|
throw err;
|
|
7331
7341
|
}
|
|
7332
7342
|
}
|
|
@@ -7335,7 +7345,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
7335
7345
|
import crypto9 from "crypto";
|
|
7336
7346
|
import { eq as eq5, and as and5, desc as desc2, inArray as inArray2, lt } from "drizzle-orm";
|
|
7337
7347
|
var MS_PER_DAY = 864e5;
|
|
7338
|
-
var
|
|
7348
|
+
var log4 = createLogger("GbpSync");
|
|
7339
7349
|
var LOCATION_CONCURRENCY = 4;
|
|
7340
7350
|
var DEFAULT_DAYS_OF_METRICS = 30;
|
|
7341
7351
|
var DEFAULT_MONTHS_OF_KEYWORDS = 12;
|
|
@@ -7396,7 +7406,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
7396
7406
|
const trendMonths = Array.from({ length: KEYWORD_TREND_MONTHS }, (_, i) => monthMinus(i + 1));
|
|
7397
7407
|
const keywordRetentionCutoff = monthKey(monthMinus(KEYWORD_HISTORY_RETENTION_MONTHS));
|
|
7398
7408
|
const { apiKey: placesApiKey, tier: placesTier, refreshIntervalDays: placesRefreshDays } = getPlacesConfig(opts.config);
|
|
7399
|
-
|
|
7409
|
+
log4.info("sync.start", { runId, projectId, locations: locationRows.length, daysOfMetrics, monthsOfKeywords });
|
|
7400
7410
|
const errors = /* @__PURE__ */ new Map();
|
|
7401
7411
|
let okCount = 0;
|
|
7402
7412
|
for (let i = 0; i < locationRows.length; i += LOCATION_CONCURRENCY) {
|
|
@@ -7448,7 +7458,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
7448
7458
|
placeToTouch = latestPlace.id;
|
|
7449
7459
|
}
|
|
7450
7460
|
} catch (placesErr) {
|
|
7451
|
-
|
|
7461
|
+
log4.warn("places.failed", { runId, location: loc.locationName, error: describeError(placesErr) });
|
|
7452
7462
|
}
|
|
7453
7463
|
}
|
|
7454
7464
|
}
|
|
@@ -7570,7 +7580,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
7570
7580
|
okCount++;
|
|
7571
7581
|
} catch (err) {
|
|
7572
7582
|
errors.set(loc.locationName, describeError(err));
|
|
7573
|
-
|
|
7583
|
+
log4.error("location.failed", { runId, location: loc.locationName, error: describeError(err) });
|
|
7574
7584
|
}
|
|
7575
7585
|
}));
|
|
7576
7586
|
}
|
|
@@ -7590,11 +7600,11 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
7590
7600
|
finishedAt
|
|
7591
7601
|
}).where(eq5(runs.id, runId)).run();
|
|
7592
7602
|
}
|
|
7593
|
-
|
|
7603
|
+
log4.info("sync.done", { runId, projectId, ok: okCount, failed: errors.size });
|
|
7594
7604
|
} catch (err) {
|
|
7595
7605
|
const errorMsg = describeError(err);
|
|
7596
7606
|
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
7597
|
-
|
|
7607
|
+
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
7598
7608
|
throw err;
|
|
7599
7609
|
}
|
|
7600
7610
|
}
|
|
@@ -8162,7 +8172,7 @@ function removeOpenAiAdsConnection(config, projectName) {
|
|
|
8162
8172
|
}
|
|
8163
8173
|
|
|
8164
8174
|
// src/ads-sync.ts
|
|
8165
|
-
var
|
|
8175
|
+
var log5 = createLogger("AdsSync");
|
|
8166
8176
|
var CAMPAIGN_INSIGHT_FIELDS = ["campaign.impressions", "campaign.clicks", "campaign.spend", "campaign.conversions", "metadata.readable_time"];
|
|
8167
8177
|
var AD_GROUP_INSIGHT_FIELDS = ["ad_group.impressions", "ad_group.clicks", "ad_group.spend", "ad_group.conversions", "metadata.readable_time"];
|
|
8168
8178
|
var CAMPAIGN_IN_PROGRESS_INSIGHT_FIELDS = CAMPAIGN_INSIGHT_FIELDS.filter((field) => field !== "campaign.conversions");
|
|
@@ -8220,7 +8230,7 @@ function toInsightUpserts(level, entityId, rows, opts) {
|
|
|
8220
8230
|
const upserts = [];
|
|
8221
8231
|
for (const row of rows) {
|
|
8222
8232
|
if (!row.readable_time) {
|
|
8223
|
-
|
|
8233
|
+
log5.warn("insights.row-missing-date", { level, entityId, rowId: row.id });
|
|
8224
8234
|
continue;
|
|
8225
8235
|
}
|
|
8226
8236
|
upserts.push({
|
|
@@ -8261,7 +8271,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
8261
8271
|
throw new Error('No OpenAI Ads API key in the local Canonry config. Run "canonry ads connect" first.');
|
|
8262
8272
|
}
|
|
8263
8273
|
const apiKey = cfgConn.apiKey;
|
|
8264
|
-
|
|
8274
|
+
log5.info("sync.start", { runId, projectId, adAccountId: connRow.adAccountId });
|
|
8265
8275
|
const account = await getAdAccount(apiKey);
|
|
8266
8276
|
const campaigns = await listCampaigns(apiKey);
|
|
8267
8277
|
const insightAnchor = /* @__PURE__ */ new Date();
|
|
@@ -8304,7 +8314,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
8304
8314
|
}
|
|
8305
8315
|
} catch (err) {
|
|
8306
8316
|
errors.set(campaign.name, describeError(err));
|
|
8307
|
-
|
|
8317
|
+
log5.error("campaign.failed", { runId, campaignId: campaign.id, error: describeError(err) });
|
|
8308
8318
|
}
|
|
8309
8319
|
}
|
|
8310
8320
|
const conversionTrackingConfigured = campaigns.some(
|
|
@@ -8424,11 +8434,11 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
8424
8434
|
finishedAt
|
|
8425
8435
|
}).where(eq6(runs.id, runId)).run();
|
|
8426
8436
|
}
|
|
8427
|
-
|
|
8437
|
+
log5.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
8428
8438
|
} catch (err) {
|
|
8429
8439
|
const errorMsg = describeError(err);
|
|
8430
8440
|
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
8431
|
-
|
|
8441
|
+
log5.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
8432
8442
|
throw err;
|
|
8433
8443
|
}
|
|
8434
8444
|
}
|
|
@@ -8438,7 +8448,7 @@ import crypto12 from "crypto";
|
|
|
8438
8448
|
import { eq as eq7 } from "drizzle-orm";
|
|
8439
8449
|
|
|
8440
8450
|
// src/sitemap-parser.ts
|
|
8441
|
-
var
|
|
8451
|
+
var log6 = createLogger("SitemapParser");
|
|
8442
8452
|
var LOC_REGEX = /<loc>([^<]+)<\/loc>/gi;
|
|
8443
8453
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
8444
8454
|
async function validateSitemapUrl(url) {
|
|
@@ -8480,7 +8490,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
8480
8490
|
res = await fetch(url);
|
|
8481
8491
|
} catch (err) {
|
|
8482
8492
|
if (!isChild) throw err;
|
|
8483
|
-
|
|
8493
|
+
log6.warn("child-sitemap.fetch-failed", {
|
|
8484
8494
|
url,
|
|
8485
8495
|
error: describeError(err)
|
|
8486
8496
|
});
|
|
@@ -8490,7 +8500,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
8490
8500
|
if (!isChild) {
|
|
8491
8501
|
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
8492
8502
|
}
|
|
8493
|
-
|
|
8503
|
+
log6.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
8494
8504
|
return;
|
|
8495
8505
|
}
|
|
8496
8506
|
let xml;
|
|
@@ -8498,7 +8508,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
8498
8508
|
xml = await readSitemapBody(res);
|
|
8499
8509
|
} catch (err) {
|
|
8500
8510
|
if (!isChild) throw err;
|
|
8501
|
-
|
|
8511
|
+
log6.warn("child-sitemap.parse-failed", {
|
|
8502
8512
|
url,
|
|
8503
8513
|
error: describeError(err)
|
|
8504
8514
|
});
|
|
@@ -8644,7 +8654,7 @@ async function inspectUrlsPaced(urls, cb, deps) {
|
|
|
8644
8654
|
}
|
|
8645
8655
|
|
|
8646
8656
|
// src/gsc-inspect-sitemap.ts
|
|
8647
|
-
var
|
|
8657
|
+
var log7 = createLogger("InspectSitemap");
|
|
8648
8658
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
8649
8659
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8650
8660
|
db.update(runs).set({ status: "running", startedAt: now }).where(eq7(runs.id, runId)).run();
|
|
@@ -8678,16 +8688,16 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
8678
8688
|
saveConfigPatch(opts.config);
|
|
8679
8689
|
}
|
|
8680
8690
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
8681
|
-
|
|
8691
|
+
log7.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
8682
8692
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
8683
|
-
|
|
8693
|
+
log7.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
8684
8694
|
if (urls.length === 0) {
|
|
8685
8695
|
throw new Error("No URLs found in sitemap");
|
|
8686
8696
|
}
|
|
8687
8697
|
const skipped = Math.max(0, urls.length - INSPECT_SWEEP_MAX_URLS);
|
|
8688
8698
|
const targetUrls = skipped > 0 ? urls.slice(0, INSPECT_SWEEP_MAX_URLS) : urls;
|
|
8689
8699
|
if (skipped > 0) {
|
|
8690
|
-
|
|
8700
|
+
log7.warn("sitemap.over-budget", {
|
|
8691
8701
|
runId,
|
|
8692
8702
|
projectId,
|
|
8693
8703
|
sitemapUrls: urls.length,
|
|
@@ -8725,10 +8735,10 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
8725
8735
|
inspectedAt,
|
|
8726
8736
|
createdAt: inspectedAt
|
|
8727
8737
|
}).run();
|
|
8728
|
-
|
|
8738
|
+
log7.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${index + 1}/${urls.length}` });
|
|
8729
8739
|
},
|
|
8730
8740
|
onError: (pageUrl, err) => {
|
|
8731
|
-
|
|
8741
|
+
log7.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: describeError(err) });
|
|
8732
8742
|
}
|
|
8733
8743
|
},
|
|
8734
8744
|
{
|
|
@@ -8739,8 +8749,8 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
8739
8749
|
// coverage refresh is exactly the overlap it fails on.
|
|
8740
8750
|
rateGateKey: `gsc:${propertyId}`,
|
|
8741
8751
|
log: {
|
|
8742
|
-
info: (action, ctx) =>
|
|
8743
|
-
error: (action, ctx) =>
|
|
8752
|
+
info: (action, ctx) => log7.info(action, { runId, projectId, ...ctx }),
|
|
8753
|
+
error: (action, ctx) => log7.error(action, { runId, projectId, ...ctx })
|
|
8744
8754
|
}
|
|
8745
8755
|
}
|
|
8746
8756
|
);
|
|
@@ -8756,11 +8766,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
8756
8766
|
const attempted = targetUrls.length;
|
|
8757
8767
|
const status = skipped > 0 || errors > 0 && inspected > 0 ? "partial" : errors === attempted ? "failed" : "completed";
|
|
8758
8768
|
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
8759
|
-
|
|
8769
|
+
log7.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
8760
8770
|
} catch (err) {
|
|
8761
8771
|
const errorMsg = describeError(err);
|
|
8762
8772
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
8763
|
-
|
|
8773
|
+
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
8764
8774
|
throw err;
|
|
8765
8775
|
}
|
|
8766
8776
|
}
|
|
@@ -8768,7 +8778,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
8768
8778
|
// src/bing-inspect-sitemap.ts
|
|
8769
8779
|
import crypto13 from "crypto";
|
|
8770
8780
|
import { eq as eq8, desc as desc3 } from "drizzle-orm";
|
|
8771
|
-
var
|
|
8781
|
+
var log8 = createLogger("BingInspectSitemap");
|
|
8772
8782
|
var BING_INSPECT_SPACING_MS = 2e3;
|
|
8773
8783
|
function parseBingDate(value) {
|
|
8774
8784
|
if (!value) return null;
|
|
@@ -8800,16 +8810,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8800
8810
|
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
8801
8811
|
}
|
|
8802
8812
|
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
8803
|
-
|
|
8813
|
+
log8.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
8804
8814
|
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
8805
|
-
|
|
8815
|
+
log8.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
8806
8816
|
if (sitemapUrls.length === 0) {
|
|
8807
8817
|
throw new Error("No URLs found in sitemap");
|
|
8808
8818
|
}
|
|
8809
8819
|
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq8(bingUrlInspections.projectId, projectId)).all();
|
|
8810
8820
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
8811
8821
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
8812
|
-
|
|
8822
|
+
log8.info("sitemap.diff", {
|
|
8813
8823
|
runId,
|
|
8814
8824
|
projectId,
|
|
8815
8825
|
sitemapTotal: sitemapUrls.length,
|
|
@@ -8824,9 +8834,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8824
8834
|
blockedUrls.add(issue.Url);
|
|
8825
8835
|
}
|
|
8826
8836
|
}
|
|
8827
|
-
|
|
8837
|
+
log8.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
8828
8838
|
} catch (err) {
|
|
8829
|
-
|
|
8839
|
+
log8.warn("crawl-issues.lookup-failed", {
|
|
8830
8840
|
runId,
|
|
8831
8841
|
projectId,
|
|
8832
8842
|
error: describeError(err)
|
|
@@ -8836,7 +8846,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8836
8846
|
const skipped = Math.max(0, sitemapUrls.length - INSPECT_SWEEP_MAX_URLS);
|
|
8837
8847
|
const targetUrls = skipped > 0 ? sitemapUrls.slice(0, INSPECT_SWEEP_MAX_URLS) : sitemapUrls;
|
|
8838
8848
|
if (skipped > 0) {
|
|
8839
|
-
|
|
8849
|
+
log8.warn("sitemap.over-budget", {
|
|
8840
8850
|
runId,
|
|
8841
8851
|
projectId,
|
|
8842
8852
|
sitemapUrls: sitemapUrls.length,
|
|
@@ -8886,11 +8896,11 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8886
8896
|
discoveryDate
|
|
8887
8897
|
}).run();
|
|
8888
8898
|
inspected++;
|
|
8889
|
-
|
|
8899
|
+
log8.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${targetUrls.length}` });
|
|
8890
8900
|
},
|
|
8891
8901
|
onError: (pageUrl, err) => {
|
|
8892
8902
|
errors++;
|
|
8893
|
-
|
|
8903
|
+
log8.error("inspect.url-failed", {
|
|
8894
8904
|
runId,
|
|
8895
8905
|
projectId,
|
|
8896
8906
|
url: pageUrl,
|
|
@@ -8912,12 +8922,12 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8912
8922
|
// attempts = 20 requests for one throttled URL, so the breaker's
|
|
8913
8923
|
// five-failure budget spent ~100 requests and inspected nothing.
|
|
8914
8924
|
maxRetries: 0,
|
|
8915
|
-
log:
|
|
8925
|
+
log: log8,
|
|
8916
8926
|
...opts.pacedDeps
|
|
8917
8927
|
}
|
|
8918
8928
|
);
|
|
8919
8929
|
if (outcome.aborted) {
|
|
8920
|
-
|
|
8930
|
+
log8.error("inspect.circuit-break", {
|
|
8921
8931
|
runId,
|
|
8922
8932
|
projectId,
|
|
8923
8933
|
inspected,
|
|
@@ -8976,7 +8986,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8976
8986
|
const status = errors > 0 && errors >= targetUrls.length ? RunStatuses.failed : errors > 0 || outcome.aborted ? RunStatuses.partial : RunStatuses.completed;
|
|
8977
8987
|
const degradedReason = status === RunStatuses.completed ? null : outcome.aborted ? `Stopped early after sustained failures; ${targetUrls.length - inspected} of ${targetUrls.length} pages were not inspected.` : `${errors} of ${targetUrls.length} pages could not be inspected.`;
|
|
8978
8988
|
db.update(runs).set({ status, error: degradedReason, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq8(runs.id, runId)).run();
|
|
8979
|
-
|
|
8989
|
+
log8.info("inspect.completed", {
|
|
8980
8990
|
runId,
|
|
8981
8991
|
projectId,
|
|
8982
8992
|
inspected,
|
|
@@ -8990,7 +9000,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8990
9000
|
} catch (err) {
|
|
8991
9001
|
const errorMsg = describeError(err);
|
|
8992
9002
|
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq8(runs.id, runId)).run();
|
|
8993
|
-
|
|
9003
|
+
log8.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
8994
9004
|
throw err;
|
|
8995
9005
|
}
|
|
8996
9006
|
}
|
|
@@ -8998,7 +9008,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
8998
9008
|
// src/coverage-refresh.ts
|
|
8999
9009
|
import crypto14 from "crypto";
|
|
9000
9010
|
import { and as and6, desc as desc4, eq as eq9, inArray as inArray3 } from "drizzle-orm";
|
|
9001
|
-
var
|
|
9011
|
+
var log9 = createLogger("CoverageRefresh");
|
|
9002
9012
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
9003
9013
|
var COVERAGE_REFRESH_MANUAL_MIN_INTERVAL_MS = 2 * 60 * 1e3;
|
|
9004
9014
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
@@ -9028,7 +9038,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
9028
9038
|
)
|
|
9029
9039
|
).orderBy(desc4(runs.createdAt)).limit(1).get();
|
|
9030
9040
|
if (inFlight2) {
|
|
9031
|
-
|
|
9041
|
+
log9.info("skip.in-flight", { projectId });
|
|
9032
9042
|
return null;
|
|
9033
9043
|
}
|
|
9034
9044
|
const windowMs = opts.userInitiated ? COVERAGE_REFRESH_MANUAL_MIN_INTERVAL_MS : COVERAGE_REFRESH_MIN_INTERVAL_MS;
|
|
@@ -9042,7 +9052,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
9042
9052
|
if (recent) {
|
|
9043
9053
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
9044
9054
|
if (Number.isFinite(ageMs) && ageMs < windowMs) {
|
|
9045
|
-
|
|
9055
|
+
log9.info("skip.recent", { projectId, ageMs, windowMs, userInitiated: opts.userInitiated === true });
|
|
9046
9056
|
return null;
|
|
9047
9057
|
}
|
|
9048
9058
|
}
|
|
@@ -9055,11 +9065,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
9055
9065
|
trigger: RunTriggers.scheduled,
|
|
9056
9066
|
createdAt: new Date(nowMs).toISOString()
|
|
9057
9067
|
}).run();
|
|
9058
|
-
|
|
9068
|
+
log9.info("refresh.start", { projectId, runId });
|
|
9059
9069
|
try {
|
|
9060
9070
|
await deps.executeInspectSitemap(db, runId, projectId, { config });
|
|
9061
9071
|
} catch (err) {
|
|
9062
|
-
|
|
9072
|
+
log9.error("refresh.failed", {
|
|
9063
9073
|
projectId,
|
|
9064
9074
|
runId,
|
|
9065
9075
|
error: describeError(err)
|
|
@@ -9072,7 +9082,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
9072
9082
|
import crypto15 from "crypto";
|
|
9073
9083
|
import path4 from "path";
|
|
9074
9084
|
import { and as and7, eq as eq10, sql as sql4 } from "drizzle-orm";
|
|
9075
|
-
var
|
|
9085
|
+
var log10 = createLogger("CommonCrawlSync");
|
|
9076
9086
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
9077
9087
|
function defaultDeps2() {
|
|
9078
9088
|
return {
|
|
@@ -9198,7 +9208,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
9198
9208
|
updatedAt: finishedAt,
|
|
9199
9209
|
error: null
|
|
9200
9210
|
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
9201
|
-
|
|
9211
|
+
log10.info("sync.completed", {
|
|
9202
9212
|
syncId,
|
|
9203
9213
|
release,
|
|
9204
9214
|
projectsProcessed: allProjects.length,
|
|
@@ -9210,7 +9220,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
9210
9220
|
try {
|
|
9211
9221
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
9212
9222
|
} catch (err) {
|
|
9213
|
-
|
|
9223
|
+
log10.error("auto-extract.enqueue-failed", {
|
|
9214
9224
|
syncId,
|
|
9215
9225
|
release,
|
|
9216
9226
|
projectId: p.id,
|
|
@@ -9228,7 +9238,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
9228
9238
|
phaseDetail: null,
|
|
9229
9239
|
updatedAt: finishedAt
|
|
9230
9240
|
}).where(eq10(ccReleaseSyncs.id, syncId)).run();
|
|
9231
|
-
|
|
9241
|
+
log10.error("sync.failed", { syncId, release, error: errorMsg });
|
|
9232
9242
|
throw err;
|
|
9233
9243
|
}
|
|
9234
9244
|
}
|
|
@@ -9264,7 +9274,7 @@ function computeSummary(rows) {
|
|
|
9264
9274
|
import crypto16 from "crypto";
|
|
9265
9275
|
import fs3 from "fs";
|
|
9266
9276
|
import { and as and8, desc as desc5, eq as eq11 } from "drizzle-orm";
|
|
9267
|
-
var
|
|
9277
|
+
var log11 = createLogger("BacklinkExtract");
|
|
9268
9278
|
function defaultDeps3() {
|
|
9269
9279
|
return {
|
|
9270
9280
|
queryBacklinks,
|
|
@@ -9356,7 +9366,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
9356
9366
|
});
|
|
9357
9367
|
const finishedAt = deps.now().toISOString();
|
|
9358
9368
|
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
9359
|
-
|
|
9369
|
+
log11.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
9360
9370
|
} catch (err) {
|
|
9361
9371
|
const errorMsg = describeError(err);
|
|
9362
9372
|
const finishedAt = deps.now().toISOString();
|
|
@@ -9365,7 +9375,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
9365
9375
|
error: errorMsg,
|
|
9366
9376
|
finishedAt
|
|
9367
9377
|
}).where(eq11(runs.id, runId)).run();
|
|
9368
|
-
|
|
9378
|
+
log11.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
9369
9379
|
throw err;
|
|
9370
9380
|
}
|
|
9371
9381
|
}
|
|
@@ -9376,7 +9386,7 @@ function computeSummary2(rows) {
|
|
|
9376
9386
|
// src/discovery-run.ts
|
|
9377
9387
|
import crypto17 from "crypto";
|
|
9378
9388
|
import { and as and9, eq as eq12 } from "drizzle-orm";
|
|
9379
|
-
var
|
|
9389
|
+
var log12 = createLogger("DiscoveryRun");
|
|
9380
9390
|
var DEFAULT_SEED_COUNT = 30;
|
|
9381
9391
|
var EMBED_RETRY_MAX_RETRIES = 3;
|
|
9382
9392
|
var EMBED_RETRY_MAX_DELAY_MS = 1e4;
|
|
@@ -9409,7 +9419,7 @@ async function embedWithRetry(fn, opts = {}) {
|
|
|
9409
9419
|
maxDelayMs: EMBED_RETRY_MAX_DELAY_MS,
|
|
9410
9420
|
sleep: opts.sleep,
|
|
9411
9421
|
onRetry: opts.onRetry ?? (({ attempt, delayMs }) => {
|
|
9412
|
-
|
|
9422
|
+
log12.warn("discovery.embed.retry", { attempt, delayMs });
|
|
9413
9423
|
})
|
|
9414
9424
|
}
|
|
9415
9425
|
);
|
|
@@ -9460,7 +9470,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
9460
9470
|
result
|
|
9461
9471
|
});
|
|
9462
9472
|
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq12(runs.id, opts.runId)).run();
|
|
9463
|
-
|
|
9473
|
+
log12.info("discovery.completed", {
|
|
9464
9474
|
runId: opts.runId,
|
|
9465
9475
|
sessionId: opts.sessionId,
|
|
9466
9476
|
buckets: result.buckets,
|
|
@@ -9468,7 +9478,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
9468
9478
|
});
|
|
9469
9479
|
} catch (err) {
|
|
9470
9480
|
const errorMsg = describeError(err);
|
|
9471
|
-
|
|
9481
|
+
log12.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
9472
9482
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
9473
9483
|
opts.db.update(runs).set({
|
|
9474
9484
|
status: RunStatuses.failed,
|
|
@@ -10685,7 +10695,7 @@ function classifySiteCrawlTemplateLinks(db, scope, pagesFetched, placementRulese
|
|
|
10685
10695
|
}
|
|
10686
10696
|
|
|
10687
10697
|
// src/execute-site-audit.ts
|
|
10688
|
-
var
|
|
10698
|
+
var log13 = createLogger("SiteAudit");
|
|
10689
10699
|
var SiteAuditCancelledError = class extends Error {
|
|
10690
10700
|
name = "SiteAuditCancelledError";
|
|
10691
10701
|
};
|
|
@@ -11135,7 +11145,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11135
11145
|
});
|
|
11136
11146
|
const crawlRootUrl = resolvedRoot.effectiveUrl;
|
|
11137
11147
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
11138
|
-
|
|
11148
|
+
log13.info("start", {
|
|
11139
11149
|
runId,
|
|
11140
11150
|
projectId,
|
|
11141
11151
|
homepageUrl,
|
|
@@ -11167,7 +11177,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11167
11177
|
crawlSummary.pagesFetched,
|
|
11168
11178
|
placementRulesetVersion
|
|
11169
11179
|
);
|
|
11170
|
-
|
|
11180
|
+
log13.info("template-links", {
|
|
11171
11181
|
runId,
|
|
11172
11182
|
projectId,
|
|
11173
11183
|
detection: templateLinks.detection,
|
|
@@ -11193,7 +11203,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11193
11203
|
try {
|
|
11194
11204
|
persistSiteCrawlGraphLayout(db, { projectId, runId, attemptId }, graphLayout, finishedAt);
|
|
11195
11205
|
} catch (error) {
|
|
11196
|
-
|
|
11206
|
+
log13.warn("graph-layout.persist-failed", {
|
|
11197
11207
|
runId,
|
|
11198
11208
|
projectId,
|
|
11199
11209
|
error: describeError(error)
|
|
@@ -11211,7 +11221,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11211
11221
|
edges: []
|
|
11212
11222
|
}, finishedAt);
|
|
11213
11223
|
} catch (markerError) {
|
|
11214
|
-
|
|
11224
|
+
log13.warn("graph-layout.marker-persist-failed", {
|
|
11215
11225
|
runId,
|
|
11216
11226
|
projectId,
|
|
11217
11227
|
error: describeError(markerError)
|
|
@@ -11331,7 +11341,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11331
11341
|
try {
|
|
11332
11342
|
deleteSiteCrawlGraphLayout(db, { projectId, runId, attemptId });
|
|
11333
11343
|
} catch (cleanupError) {
|
|
11334
|
-
|
|
11344
|
+
log13.warn("graph-layout.cleanup-failed", {
|
|
11335
11345
|
runId,
|
|
11336
11346
|
projectId,
|
|
11337
11347
|
error: describeError(cleanupError)
|
|
@@ -11341,7 +11351,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11341
11351
|
if (current?.status === "cancelled") throw new SiteAuditCancelledError(`Site audit cancelled before publication: ${runId}`);
|
|
11342
11352
|
throw new Error(`Site audit run was no longer running before publication: ${runId}`);
|
|
11343
11353
|
}
|
|
11344
|
-
|
|
11354
|
+
log13.info("completed", { runId, projectId, status: terminalStatus, complete: crawlSummary.complete, pages: crawlSummary.pagesObserved });
|
|
11345
11355
|
} catch (error) {
|
|
11346
11356
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11347
11357
|
const cancelled = isCancelled(error, opts.signal);
|
|
@@ -11359,7 +11369,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
11359
11369
|
finishedAt
|
|
11360
11370
|
}).where(and12(eq15(runs.id, runId), eq15(runs.status, "running"))).run();
|
|
11361
11371
|
});
|
|
11362
|
-
|
|
11372
|
+
log13.error("failed", { runId, projectId, cancelled, error: message });
|
|
11363
11373
|
throw error;
|
|
11364
11374
|
}
|
|
11365
11375
|
}
|
|
@@ -11829,7 +11839,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
11829
11839
|
return result;
|
|
11830
11840
|
}
|
|
11831
11841
|
async function backfillInsightsCommand(project, opts) {
|
|
11832
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
11842
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-2E7F6I7I.js");
|
|
11833
11843
|
const config = loadConfig();
|
|
11834
11844
|
const db = createClient(config.database);
|
|
11835
11845
|
migrate(db);
|
|
@@ -12408,7 +12418,7 @@ var ProviderRegistry = class {
|
|
|
12408
12418
|
import crypto20 from "crypto";
|
|
12409
12419
|
import cron from "node-cron";
|
|
12410
12420
|
import { and as and14, eq as eq17, inArray as inArray6, notExists, sql as sql8 } from "drizzle-orm";
|
|
12411
|
-
var
|
|
12421
|
+
var log14 = createLogger("Scheduler");
|
|
12412
12422
|
var DEFAULT_HEALTH_CRON = "0 */6 * * *";
|
|
12413
12423
|
function taskKey(projectId, kind) {
|
|
12414
12424
|
return `${projectId}::${kind}`;
|
|
@@ -12460,7 +12470,7 @@ var Scheduler = class {
|
|
|
12460
12470
|
updatedAt: now
|
|
12461
12471
|
}).run();
|
|
12462
12472
|
}
|
|
12463
|
-
|
|
12473
|
+
log14.info("health-schedule.seeded", { projectCount: projectsWithoutHealth.length, cron: DEFAULT_HEALTH_CRON });
|
|
12464
12474
|
}
|
|
12465
12475
|
/**
|
|
12466
12476
|
* Record each project's current query set as basket revision 1 if it has
|
|
@@ -12483,10 +12493,10 @@ var Scheduler = class {
|
|
|
12483
12493
|
const current = ensureCurrentQueryBasketRevision(this.db, project.id);
|
|
12484
12494
|
if (current !== null && current.revision !== previous) minted += 1;
|
|
12485
12495
|
} catch (err) {
|
|
12486
|
-
|
|
12496
|
+
log14.warn("query-basket.mint-failed", { projectId: project.id, err: String(err) });
|
|
12487
12497
|
}
|
|
12488
12498
|
}
|
|
12489
|
-
if (minted > 0)
|
|
12499
|
+
if (minted > 0) log14.info("query-basket.ensured", { projectCount: minted });
|
|
12490
12500
|
}
|
|
12491
12501
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
12492
12502
|
start() {
|
|
@@ -12497,11 +12507,11 @@ var Scheduler = class {
|
|
|
12497
12507
|
const missedRunAt = schedule.nextRunAt;
|
|
12498
12508
|
this.registerCronTask(schedule);
|
|
12499
12509
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
12500
|
-
|
|
12510
|
+
log14.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
12501
12511
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
12502
12512
|
}
|
|
12503
12513
|
}
|
|
12504
|
-
|
|
12514
|
+
log14.info("started", { scheduleCount: allSchedules.length });
|
|
12505
12515
|
}
|
|
12506
12516
|
/** Stop all cron tasks for graceful shutdown. */
|
|
12507
12517
|
stop() {
|
|
@@ -12545,13 +12555,13 @@ var Scheduler = class {
|
|
|
12545
12555
|
stopTask(key, task, verb) {
|
|
12546
12556
|
void task.stop();
|
|
12547
12557
|
void task.destroy();
|
|
12548
|
-
|
|
12558
|
+
log14.info(`task.${verb.toLowerCase()}`, { key });
|
|
12549
12559
|
}
|
|
12550
12560
|
registerCronTask(schedule) {
|
|
12551
12561
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
12552
12562
|
const kind = schedule.kind;
|
|
12553
12563
|
if (!cron.validate(cronExpr)) {
|
|
12554
|
-
|
|
12564
|
+
log14.error("cron.invalid", { projectId, kind, cronExpr });
|
|
12555
12565
|
return;
|
|
12556
12566
|
}
|
|
12557
12567
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -12565,32 +12575,32 @@ var Scheduler = class {
|
|
|
12565
12575
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12566
12576
|
}).where(eq17(schedules.id, scheduleId)).run();
|
|
12567
12577
|
const label = schedule.preset ?? cronExpr;
|
|
12568
|
-
|
|
12578
|
+
log14.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
12569
12579
|
}
|
|
12570
12580
|
triggerRun(scheduleId, projectId, kind) {
|
|
12571
12581
|
try {
|
|
12572
12582
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12573
12583
|
const currentSchedule = this.db.select().from(schedules).where(eq17(schedules.id, scheduleId)).get();
|
|
12574
12584
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
12575
|
-
|
|
12585
|
+
log14.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
12576
12586
|
this.remove(projectId, kind);
|
|
12577
12587
|
return;
|
|
12578
12588
|
}
|
|
12579
12589
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
12580
12590
|
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
12581
12591
|
if (!project) {
|
|
12582
|
-
|
|
12592
|
+
log14.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
12583
12593
|
this.remove(projectId, kind);
|
|
12584
12594
|
return;
|
|
12585
12595
|
}
|
|
12586
12596
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
12587
12597
|
const sourceId = currentSchedule.sourceId;
|
|
12588
12598
|
if (!sourceId) {
|
|
12589
|
-
|
|
12599
|
+
log14.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
12590
12600
|
return;
|
|
12591
12601
|
}
|
|
12592
12602
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
12593
|
-
|
|
12603
|
+
log14.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
12594
12604
|
return;
|
|
12595
12605
|
}
|
|
12596
12606
|
this.db.update(schedules).set({
|
|
@@ -12598,13 +12608,13 @@ var Scheduler = class {
|
|
|
12598
12608
|
nextRunAt,
|
|
12599
12609
|
updatedAt: now
|
|
12600
12610
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12601
|
-
|
|
12611
|
+
log14.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
12602
12612
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
12603
12613
|
return;
|
|
12604
12614
|
}
|
|
12605
12615
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
12606
12616
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
12607
|
-
|
|
12617
|
+
log14.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
12608
12618
|
return;
|
|
12609
12619
|
}
|
|
12610
12620
|
const runId2 = crypto20.randomUUID();
|
|
@@ -12621,13 +12631,13 @@ var Scheduler = class {
|
|
|
12621
12631
|
nextRunAt,
|
|
12622
12632
|
updatedAt: now
|
|
12623
12633
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12624
|
-
|
|
12634
|
+
log14.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
12625
12635
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
12626
12636
|
return;
|
|
12627
12637
|
}
|
|
12628
12638
|
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
12629
12639
|
if (!this.callbacks.onAdsSyncRequested) {
|
|
12630
|
-
|
|
12640
|
+
log14.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
12631
12641
|
return;
|
|
12632
12642
|
}
|
|
12633
12643
|
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
@@ -12636,7 +12646,7 @@ var Scheduler = class {
|
|
|
12636
12646
|
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
12637
12647
|
)).get();
|
|
12638
12648
|
if (activeAdsRun) {
|
|
12639
|
-
|
|
12649
|
+
log14.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
12640
12650
|
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12641
12651
|
return;
|
|
12642
12652
|
}
|
|
@@ -12654,13 +12664,13 @@ var Scheduler = class {
|
|
|
12654
12664
|
nextRunAt,
|
|
12655
12665
|
updatedAt: now
|
|
12656
12666
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12657
|
-
|
|
12667
|
+
log14.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
12658
12668
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
12659
12669
|
return;
|
|
12660
12670
|
}
|
|
12661
12671
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
12662
12672
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
12663
|
-
|
|
12673
|
+
log14.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
12664
12674
|
return;
|
|
12665
12675
|
}
|
|
12666
12676
|
this.db.update(schedules).set({
|
|
@@ -12668,13 +12678,13 @@ var Scheduler = class {
|
|
|
12668
12678
|
nextRunAt,
|
|
12669
12679
|
updatedAt: now
|
|
12670
12680
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12671
|
-
|
|
12681
|
+
log14.info("data-refresh.triggered", { projectName: project.name });
|
|
12672
12682
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
12673
12683
|
return;
|
|
12674
12684
|
}
|
|
12675
12685
|
if (kind === SchedulableRunKinds.doctor) {
|
|
12676
12686
|
if (!this.callbacks.onDoctorRequested) {
|
|
12677
|
-
|
|
12687
|
+
log14.warn("doctor.no-callback", { scheduleId, projectId, msg: "host did not register onDoctorRequested" });
|
|
12678
12688
|
return;
|
|
12679
12689
|
}
|
|
12680
12690
|
this.db.update(schedules).set({
|
|
@@ -12682,13 +12692,13 @@ var Scheduler = class {
|
|
|
12682
12692
|
nextRunAt,
|
|
12683
12693
|
updatedAt: now
|
|
12684
12694
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12685
|
-
|
|
12695
|
+
log14.info("doctor.triggered", { projectName: project.name });
|
|
12686
12696
|
this.callbacks.onDoctorRequested(project.name);
|
|
12687
12697
|
return;
|
|
12688
12698
|
}
|
|
12689
12699
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
12690
12700
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
12691
|
-
|
|
12701
|
+
log14.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
12692
12702
|
return;
|
|
12693
12703
|
}
|
|
12694
12704
|
this.db.update(schedules).set({
|
|
@@ -12696,13 +12706,13 @@ var Scheduler = class {
|
|
|
12696
12706
|
nextRunAt,
|
|
12697
12707
|
updatedAt: now
|
|
12698
12708
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12699
|
-
|
|
12709
|
+
log14.info("backlinks-sync.triggered", { projectName: project.name });
|
|
12700
12710
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
12701
12711
|
return;
|
|
12702
12712
|
}
|
|
12703
12713
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
12704
12714
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
12705
|
-
|
|
12715
|
+
log14.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
12706
12716
|
return;
|
|
12707
12717
|
}
|
|
12708
12718
|
const active = this.db.select({ id: runs.id }).from(runs).where(and14(
|
|
@@ -12711,7 +12721,7 @@ var Scheduler = class {
|
|
|
12711
12721
|
inArray6(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
12712
12722
|
)).get();
|
|
12713
12723
|
if (active) {
|
|
12714
|
-
|
|
12724
|
+
log14.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
12715
12725
|
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12716
12726
|
return;
|
|
12717
12727
|
}
|
|
@@ -12739,7 +12749,7 @@ var Scheduler = class {
|
|
|
12739
12749
|
nextRunAt,
|
|
12740
12750
|
updatedAt: now
|
|
12741
12751
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12742
|
-
|
|
12752
|
+
log14.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
12743
12753
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
12744
12754
|
return;
|
|
12745
12755
|
}
|
|
@@ -12748,7 +12758,7 @@ var Scheduler = class {
|
|
|
12748
12758
|
if (project.defaultLocation) {
|
|
12749
12759
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
12750
12760
|
if (!loc) {
|
|
12751
|
-
|
|
12761
|
+
log14.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
12752
12762
|
return;
|
|
12753
12763
|
}
|
|
12754
12764
|
resolvedLocation = loc;
|
|
@@ -12767,7 +12777,7 @@ var Scheduler = class {
|
|
|
12767
12777
|
providerModels: this.callbacks.getEffectiveProviderModels?.()
|
|
12768
12778
|
});
|
|
12769
12779
|
if (queueResult.conflict) {
|
|
12770
|
-
|
|
12780
|
+
log14.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
12771
12781
|
this.db.update(schedules).set({
|
|
12772
12782
|
nextRunAt,
|
|
12773
12783
|
updatedAt: now
|
|
@@ -12780,16 +12790,16 @@ var Scheduler = class {
|
|
|
12780
12790
|
nextRunAt,
|
|
12781
12791
|
updatedAt: now
|
|
12782
12792
|
}).where(eq17(schedules.id, currentSchedule.id)).run();
|
|
12783
|
-
|
|
12793
|
+
log14.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
12784
12794
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
12785
12795
|
} catch (err) {
|
|
12786
|
-
|
|
12796
|
+
log14.error("trigger.error", { scheduleId, projectId, kind, error: describeError(err) });
|
|
12787
12797
|
}
|
|
12788
12798
|
}
|
|
12789
12799
|
};
|
|
12790
12800
|
|
|
12791
12801
|
// src/data-refresh.ts
|
|
12792
|
-
var
|
|
12802
|
+
var log15 = createLogger("DataRefresh");
|
|
12793
12803
|
async function refreshAllIntegrations(client, projectName) {
|
|
12794
12804
|
const integrations = [
|
|
12795
12805
|
{ name: "gsc", run: () => client.gscSync(projectName, {}) },
|
|
@@ -12806,11 +12816,11 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
12806
12816
|
results.forEach((result, idx) => {
|
|
12807
12817
|
const integration = integrations[idx].name;
|
|
12808
12818
|
if (result.status === "fulfilled") {
|
|
12809
|
-
|
|
12819
|
+
log15.info("integration.refreshed", { projectName, integration });
|
|
12810
12820
|
} else {
|
|
12811
12821
|
const reason = result.reason;
|
|
12812
12822
|
const message = describeError(reason);
|
|
12813
|
-
|
|
12823
|
+
log15.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
12814
12824
|
}
|
|
12815
12825
|
});
|
|
12816
12826
|
}
|
|
@@ -12818,7 +12828,7 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
12818
12828
|
// src/notifier.ts
|
|
12819
12829
|
import { eq as eq18, desc as desc7, and as and15, inArray as inArray7, or as or2 } from "drizzle-orm";
|
|
12820
12830
|
import crypto21 from "crypto";
|
|
12821
|
-
var
|
|
12831
|
+
var log16 = createLogger("Notifier");
|
|
12822
12832
|
var Notifier = class {
|
|
12823
12833
|
db;
|
|
12824
12834
|
serverUrl;
|
|
@@ -12828,26 +12838,26 @@ var Notifier = class {
|
|
|
12828
12838
|
}
|
|
12829
12839
|
/** Called after a run completes (success, partial, or failed). */
|
|
12830
12840
|
async onRunCompleted(runId, projectId) {
|
|
12831
|
-
|
|
12841
|
+
log16.info("run.completed", { runId, projectId });
|
|
12832
12842
|
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
12833
12843
|
if (notifs.length === 0) {
|
|
12834
|
-
|
|
12844
|
+
log16.info("notifications.none-enabled", { projectId });
|
|
12835
12845
|
return;
|
|
12836
12846
|
}
|
|
12837
|
-
|
|
12847
|
+
log16.info("notifications.found", { projectId, count: notifs.length });
|
|
12838
12848
|
const run = this.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
12839
12849
|
if (!run) {
|
|
12840
|
-
|
|
12850
|
+
log16.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
12841
12851
|
return;
|
|
12842
12852
|
}
|
|
12843
12853
|
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
12844
12854
|
if (!project) {
|
|
12845
|
-
|
|
12855
|
+
log16.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
12846
12856
|
return;
|
|
12847
12857
|
}
|
|
12848
12858
|
const transitions = this.computeTransitions(runId, projectId);
|
|
12849
12859
|
const events = [];
|
|
12850
|
-
|
|
12860
|
+
log16.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
12851
12861
|
if (run.status === "completed" || run.status === "partial") {
|
|
12852
12862
|
events.push("run.completed");
|
|
12853
12863
|
}
|
|
@@ -12863,7 +12873,7 @@ var Notifier = class {
|
|
|
12863
12873
|
if (!config.url) continue;
|
|
12864
12874
|
const subscribedEvents = config.events;
|
|
12865
12875
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
12866
|
-
|
|
12876
|
+
log16.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
12867
12877
|
if (matchingEvents.length === 0) continue;
|
|
12868
12878
|
for (const event of matchingEvents) {
|
|
12869
12879
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -12895,7 +12905,7 @@ var Notifier = class {
|
|
|
12895
12905
|
async onHealthChecked(projectId, report) {
|
|
12896
12906
|
const project = this.db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
12897
12907
|
if (!project) {
|
|
12898
|
-
|
|
12908
|
+
log16.error("project.not-found", { projectId, msg: "skipping health notification" });
|
|
12899
12909
|
return null;
|
|
12900
12910
|
}
|
|
12901
12911
|
const rank = (status2) => status2 === "fail" ? 2 : status2 === "warn" ? 1 : 0;
|
|
@@ -12942,7 +12952,7 @@ var Notifier = class {
|
|
|
12942
12952
|
this.db.update(doctorHealthState).set(observation).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
12943
12953
|
}
|
|
12944
12954
|
if (!event) {
|
|
12945
|
-
|
|
12955
|
+
log16.info("health.unchanged", { projectId, status, code });
|
|
12946
12956
|
return null;
|
|
12947
12957
|
}
|
|
12948
12958
|
const notifs = this.db.select().from(notifications).where(eq18(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
@@ -12971,7 +12981,7 @@ var Notifier = class {
|
|
|
12971
12981
|
if (delivered > 0) {
|
|
12972
12982
|
this.db.update(doctorHealthState).set({ notifiedAt: now }).where(eq18(doctorHealthState.projectId, projectId)).run();
|
|
12973
12983
|
}
|
|
12974
|
-
|
|
12984
|
+
log16.info("health.notified", { projectId, event, status, code, subscribers: notifs.length, delivered });
|
|
12975
12985
|
return event;
|
|
12976
12986
|
}
|
|
12977
12987
|
/** Dispatch insight webhooks for critical/high severity insights after a run. */
|
|
@@ -13020,7 +13030,7 @@ var Notifier = class {
|
|
|
13020
13030
|
if (!thisRun) return [];
|
|
13021
13031
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
13022
13032
|
if (completeness.planned && !completeness.complete) {
|
|
13023
|
-
|
|
13033
|
+
log16.info("transitions.skipped-incomplete", {
|
|
13024
13034
|
runId,
|
|
13025
13035
|
executed: completeness.executed,
|
|
13026
13036
|
expected: completeness.expected
|
|
@@ -13109,23 +13119,23 @@ var Notifier = class {
|
|
|
13109
13119
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
13110
13120
|
const targetCheck = await resolveWebhookTarget(url);
|
|
13111
13121
|
if (!targetCheck.ok) {
|
|
13112
|
-
|
|
13122
|
+
log16.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
13113
13123
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
13114
13124
|
return;
|
|
13115
13125
|
}
|
|
13116
|
-
|
|
13126
|
+
log16.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
13117
13127
|
const maxRetries = 3;
|
|
13118
13128
|
const delays = [1e3, 4e3, 16e3];
|
|
13119
13129
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
13120
13130
|
try {
|
|
13121
13131
|
const response = await deliverWebhook(targetCheck.target, body, signingSecret);
|
|
13122
13132
|
if (response.status >= 200 && response.status < 300) {
|
|
13123
|
-
|
|
13133
|
+
log16.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
13124
13134
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
13125
13135
|
return;
|
|
13126
13136
|
}
|
|
13127
13137
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
13128
|
-
|
|
13138
|
+
log16.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
13129
13139
|
if (attempt === maxRetries - 1) {
|
|
13130
13140
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
13131
13141
|
}
|
|
@@ -13133,7 +13143,7 @@ var Notifier = class {
|
|
|
13133
13143
|
const errorDetail = describeError(err);
|
|
13134
13144
|
if (attempt === maxRetries - 1) {
|
|
13135
13145
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
13136
|
-
|
|
13146
|
+
log16.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
13137
13147
|
}
|
|
13138
13148
|
}
|
|
13139
13149
|
if (attempt < maxRetries - 1) {
|
|
@@ -13157,7 +13167,7 @@ var Notifier = class {
|
|
|
13157
13167
|
|
|
13158
13168
|
// src/run-coordinator.ts
|
|
13159
13169
|
import { eq as eq19 } from "drizzle-orm";
|
|
13160
|
-
var
|
|
13170
|
+
var log17 = createLogger("RunCoordinator");
|
|
13161
13171
|
var RunCoordinator = class {
|
|
13162
13172
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
13163
13173
|
this.db = db;
|
|
@@ -13175,12 +13185,12 @@ var RunCoordinator = class {
|
|
|
13175
13185
|
const runRow = this.db.select().from(runs).where(eq19(runs.id, runId)).get();
|
|
13176
13186
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
13177
13187
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
13178
|
-
|
|
13188
|
+
log17.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
13179
13189
|
return;
|
|
13180
13190
|
}
|
|
13181
13191
|
const completeness = measurementRunCompleteness(this.db, runId);
|
|
13182
13192
|
if (completeness.planned && !completeness.complete) {
|
|
13183
|
-
|
|
13193
|
+
log17.info("measurement.incomplete-skip-derived", {
|
|
13184
13194
|
runId,
|
|
13185
13195
|
projectId,
|
|
13186
13196
|
executed: completeness.executed,
|
|
@@ -13189,7 +13199,7 @@ var RunCoordinator = class {
|
|
|
13189
13199
|
try {
|
|
13190
13200
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
13191
13201
|
} catch (err) {
|
|
13192
|
-
|
|
13202
|
+
log17.error("notifier.failed", { runId, error: describeError(err) });
|
|
13193
13203
|
}
|
|
13194
13204
|
return;
|
|
13195
13205
|
}
|
|
@@ -13207,12 +13217,12 @@ var RunCoordinator = class {
|
|
|
13207
13217
|
try {
|
|
13208
13218
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
13209
13219
|
} catch (err) {
|
|
13210
|
-
|
|
13220
|
+
log17.error("insight-webhook.failed", { runId, error: describeError(err) });
|
|
13211
13221
|
}
|
|
13212
13222
|
}
|
|
13213
13223
|
}
|
|
13214
13224
|
} catch (err) {
|
|
13215
|
-
|
|
13225
|
+
log17.error("intelligence.failed", { runId, error: describeError(err) });
|
|
13216
13226
|
}
|
|
13217
13227
|
} else if (kind === RunKinds["gbp-sync"]) {
|
|
13218
13228
|
try {
|
|
@@ -13225,17 +13235,17 @@ var RunCoordinator = class {
|
|
|
13225
13235
|
try {
|
|
13226
13236
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
13227
13237
|
} catch (err) {
|
|
13228
|
-
|
|
13238
|
+
log17.error("gbp-insight-webhook.failed", { runId, error: describeError(err) });
|
|
13229
13239
|
}
|
|
13230
13240
|
}
|
|
13231
13241
|
} catch (err) {
|
|
13232
|
-
|
|
13242
|
+
log17.error("gbp-intelligence.failed", { runId, error: describeError(err) });
|
|
13233
13243
|
}
|
|
13234
13244
|
}
|
|
13235
13245
|
try {
|
|
13236
13246
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
13237
13247
|
} catch (err) {
|
|
13238
|
-
|
|
13248
|
+
log17.error("notifier.failed", { runId, error: describeError(err) });
|
|
13239
13249
|
}
|
|
13240
13250
|
if (this.onAeroEvent) {
|
|
13241
13251
|
try {
|
|
@@ -13248,7 +13258,7 @@ var RunCoordinator = class {
|
|
|
13248
13258
|
};
|
|
13249
13259
|
await this.onAeroEvent(ctx);
|
|
13250
13260
|
} catch (err) {
|
|
13251
|
-
|
|
13261
|
+
log17.error("aero.failed", { runId, error: describeError(err) });
|
|
13252
13262
|
}
|
|
13253
13263
|
}
|
|
13254
13264
|
}
|
|
@@ -14350,7 +14360,7 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
14350
14360
|
import { Type as Type4 } from "@sinclair/typebox";
|
|
14351
14361
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
14352
14362
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
14353
|
-
var
|
|
14363
|
+
var log18 = createLogger("RemoteMcp");
|
|
14354
14364
|
var AERO_EXCLUDED_MCP_TOOLS2 = /* @__PURE__ */ new Set([]);
|
|
14355
14365
|
var MAX_TOOL_RESULT_CHARS2 = 2e4;
|
|
14356
14366
|
function truncate(text) {
|
|
@@ -14403,7 +14413,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
14403
14413
|
try {
|
|
14404
14414
|
client = await connect(server);
|
|
14405
14415
|
} catch (err) {
|
|
14406
|
-
|
|
14416
|
+
log18.error("external-mcp.connect-failed", {
|
|
14407
14417
|
label,
|
|
14408
14418
|
error: describeError(err)
|
|
14409
14419
|
});
|
|
@@ -14413,7 +14423,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
14413
14423
|
try {
|
|
14414
14424
|
listed = await client.listTools();
|
|
14415
14425
|
} catch (err) {
|
|
14416
|
-
|
|
14426
|
+
log18.error("external-mcp.list-failed", {
|
|
14417
14427
|
label,
|
|
14418
14428
|
error: describeError(err)
|
|
14419
14429
|
});
|
|
@@ -14424,7 +14434,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
14424
14434
|
if (!isReadOnly(tool)) continue;
|
|
14425
14435
|
tools.push(adaptRemoteTool(client, tool));
|
|
14426
14436
|
}
|
|
14427
|
-
|
|
14437
|
+
log18.info("external-mcp.loaded", {
|
|
14428
14438
|
label,
|
|
14429
14439
|
discovered: listed.tools.length,
|
|
14430
14440
|
adopted: tools.length
|
|
@@ -14656,7 +14666,7 @@ async function compactMessages(args) {
|
|
|
14656
14666
|
}
|
|
14657
14667
|
|
|
14658
14668
|
// src/agent/session-registry.ts
|
|
14659
|
-
var
|
|
14669
|
+
var log19 = createLogger("SessionRegistry");
|
|
14660
14670
|
var MAX_HYDRATE_NOTES = 20;
|
|
14661
14671
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
14662
14672
|
function escapeMemoryFragment(value) {
|
|
@@ -14700,7 +14710,7 @@ var SessionRegistry = class {
|
|
|
14700
14710
|
if (!this.externalToolsPromise) {
|
|
14701
14711
|
this.externalToolsPromise = loadExternalMcpTools(this.opts.config.externalMcpServers).catch(
|
|
14702
14712
|
(err) => {
|
|
14703
|
-
|
|
14713
|
+
log19.error("external-mcp.load-failed", {
|
|
14704
14714
|
error: describeError(err)
|
|
14705
14715
|
});
|
|
14706
14716
|
return [];
|
|
@@ -14945,13 +14955,13 @@ ${lines.join("\n")}
|
|
|
14945
14955
|
agent.state.messages = result.messages;
|
|
14946
14956
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
14947
14957
|
this.save(projectName);
|
|
14948
|
-
|
|
14958
|
+
log19.info("compaction.completed", {
|
|
14949
14959
|
projectName,
|
|
14950
14960
|
removedCount: result.removedCount,
|
|
14951
14961
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
14952
14962
|
});
|
|
14953
14963
|
} catch (err) {
|
|
14954
|
-
|
|
14964
|
+
log19.error("compaction.failed", {
|
|
14955
14965
|
projectName,
|
|
14956
14966
|
error: describeError(err)
|
|
14957
14967
|
});
|
|
@@ -15052,7 +15062,7 @@ ${lines.join("\n")}
|
|
|
15052
15062
|
await agent.prompt(msgs);
|
|
15053
15063
|
this.save(projectName);
|
|
15054
15064
|
} catch (err) {
|
|
15055
|
-
|
|
15065
|
+
log19.error("drain.failed", {
|
|
15056
15066
|
projectName,
|
|
15057
15067
|
error: describeError(err)
|
|
15058
15068
|
});
|
|
@@ -15645,7 +15655,7 @@ function formatAuditFactorScore(factor) {
|
|
|
15645
15655
|
}
|
|
15646
15656
|
|
|
15647
15657
|
// src/snapshot-service.ts
|
|
15648
|
-
var
|
|
15658
|
+
var log20 = createLogger("Snapshot");
|
|
15649
15659
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
15650
15660
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
15651
15661
|
var SnapshotService = class {
|
|
@@ -15727,7 +15737,7 @@ var SnapshotService = class {
|
|
|
15727
15737
|
return mapAuditReport(report);
|
|
15728
15738
|
} catch (err) {
|
|
15729
15739
|
const message = describeError(err);
|
|
15730
|
-
|
|
15740
|
+
log20.warn("audit.failed", { homepageUrl, error: message });
|
|
15731
15741
|
return {
|
|
15732
15742
|
url: homepageUrl,
|
|
15733
15743
|
finalUrl: homepageUrl,
|
|
@@ -15756,7 +15766,7 @@ var SnapshotService = class {
|
|
|
15756
15766
|
queries: parsedQueries
|
|
15757
15767
|
};
|
|
15758
15768
|
} catch (err) {
|
|
15759
|
-
|
|
15769
|
+
log20.warn("profile.generation-failed", {
|
|
15760
15770
|
domain: ctx.domain,
|
|
15761
15771
|
provider: ctx.analysisProvider.adapter.name,
|
|
15762
15772
|
error: describeError(err)
|
|
@@ -15899,7 +15909,7 @@ var SnapshotService = class {
|
|
|
15899
15909
|
recommendedActions: uniqueStrings2(parsed.recommendedActions ?? []).slice(0, 4)
|
|
15900
15910
|
};
|
|
15901
15911
|
} catch (err) {
|
|
15902
|
-
|
|
15912
|
+
log20.warn("response.analysis-failed", {
|
|
15903
15913
|
provider: ctx.analysisProvider.adapter.name,
|
|
15904
15914
|
error: describeError(err)
|
|
15905
15915
|
});
|
|
@@ -16524,7 +16534,7 @@ async function executeGtmMarketingSync(db, runtime, runId, projectId) {
|
|
|
16524
16534
|
// src/server.ts
|
|
16525
16535
|
var _require3 = createRequire4(import.meta.url);
|
|
16526
16536
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
16527
|
-
var
|
|
16537
|
+
var log21 = createLogger("Server");
|
|
16528
16538
|
var DEFAULT_QUOTA = {
|
|
16529
16539
|
maxConcurrency: 2,
|
|
16530
16540
|
maxRequestsPerMinute: 10,
|
|
@@ -16695,7 +16705,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
16695
16705
|
}
|
|
16696
16706
|
if (migratedGoogle > 0) {
|
|
16697
16707
|
saveConfigPatch({ google: config.google });
|
|
16698
|
-
|
|
16708
|
+
log21.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
16699
16709
|
}
|
|
16700
16710
|
let migratedGa4 = 0;
|
|
16701
16711
|
for (const row of rows.ga4) {
|
|
@@ -16713,7 +16723,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
16713
16723
|
}
|
|
16714
16724
|
if (migratedGa4 > 0) {
|
|
16715
16725
|
saveConfigPatch({ ga4: config.ga4 });
|
|
16716
|
-
|
|
16726
|
+
log21.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
16717
16727
|
}
|
|
16718
16728
|
}
|
|
16719
16729
|
function isLoopbackBindHost(host) {
|
|
@@ -16892,11 +16902,11 @@ async function createServer(opts) {
|
|
|
16892
16902
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
16893
16903
|
dropLegacyCredentialColumns(opts.db);
|
|
16894
16904
|
} catch (err) {
|
|
16895
|
-
|
|
16905
|
+
log21.warn("credentials.migration.failed", {
|
|
16896
16906
|
error: describeError(err)
|
|
16897
16907
|
});
|
|
16898
16908
|
}
|
|
16899
|
-
|
|
16909
|
+
log21.info("providers.configured", {
|
|
16900
16910
|
providers: Object.keys(providers).filter((k) => {
|
|
16901
16911
|
const p = providers[k];
|
|
16902
16912
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|