@ainyc/canonry 2.5.0 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-TIHU2YXW.js → chunk-CFS35BKX.js} +125 -73
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +5 -5
|
@@ -14998,6 +14998,7 @@ import crypto21 from "crypto";
|
|
|
14998
14998
|
import { eq as eq21, and as and10 } from "drizzle-orm";
|
|
14999
14999
|
|
|
15000
15000
|
// src/sitemap-parser.ts
|
|
15001
|
+
var log3 = createLogger("SitemapParser");
|
|
15001
15002
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
15002
15003
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
15003
15004
|
var PRIVATE_IP_PATTERNS = [
|
|
@@ -15027,26 +15028,77 @@ function validateSitemapUrl(url) {
|
|
|
15027
15028
|
}
|
|
15028
15029
|
}
|
|
15029
15030
|
}
|
|
15031
|
+
async function readSitemapBody(res) {
|
|
15032
|
+
const buf = await res.arrayBuffer();
|
|
15033
|
+
const bytes = new Uint8Array(buf);
|
|
15034
|
+
const isGzipped = bytes.length >= 2 && bytes[0] === 31 && bytes[1] === 139;
|
|
15035
|
+
if (!isGzipped) {
|
|
15036
|
+
return new TextDecoder().decode(bytes);
|
|
15037
|
+
}
|
|
15038
|
+
const decompressed = new Blob([buf]).stream().pipeThrough(new DecompressionStream("gzip"));
|
|
15039
|
+
return new Response(decompressed).text();
|
|
15040
|
+
}
|
|
15030
15041
|
async function fetchAndParseSitemap(sitemapUrl) {
|
|
15031
15042
|
const urls = /* @__PURE__ */ new Set();
|
|
15032
|
-
|
|
15043
|
+
const visited = /* @__PURE__ */ new Set();
|
|
15044
|
+
await parseSitemapRecursive(
|
|
15045
|
+
sitemapUrl,
|
|
15046
|
+
urls,
|
|
15047
|
+
visited,
|
|
15048
|
+
0,
|
|
15049
|
+
/* isChild */
|
|
15050
|
+
false
|
|
15051
|
+
);
|
|
15033
15052
|
return [...urls];
|
|
15034
15053
|
}
|
|
15035
|
-
async function parseSitemapRecursive(url, urls, depth) {
|
|
15054
|
+
async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
15036
15055
|
if (depth > 3) return;
|
|
15056
|
+
if (visited.has(url)) return;
|
|
15057
|
+
visited.add(url);
|
|
15037
15058
|
validateSitemapUrl(url);
|
|
15038
|
-
|
|
15059
|
+
let res;
|
|
15060
|
+
try {
|
|
15061
|
+
res = await fetch(url);
|
|
15062
|
+
} catch (err) {
|
|
15063
|
+
if (!isChild) throw err;
|
|
15064
|
+
log3.warn("child-sitemap.fetch-failed", {
|
|
15065
|
+
url,
|
|
15066
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15067
|
+
});
|
|
15068
|
+
return;
|
|
15069
|
+
}
|
|
15039
15070
|
if (!res.ok) {
|
|
15040
|
-
|
|
15071
|
+
if (!isChild) {
|
|
15072
|
+
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
15073
|
+
}
|
|
15074
|
+
log3.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
15075
|
+
return;
|
|
15076
|
+
}
|
|
15077
|
+
let xml;
|
|
15078
|
+
try {
|
|
15079
|
+
xml = await readSitemapBody(res);
|
|
15080
|
+
} catch (err) {
|
|
15081
|
+
if (!isChild) throw err;
|
|
15082
|
+
log3.warn("child-sitemap.parse-failed", {
|
|
15083
|
+
url,
|
|
15084
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15085
|
+
});
|
|
15086
|
+
return;
|
|
15041
15087
|
}
|
|
15042
|
-
const xml = await res.text();
|
|
15043
15088
|
const sitemapEntries = xml.match(SITEMAP_TAG_REGEX);
|
|
15044
15089
|
if (sitemapEntries) {
|
|
15045
15090
|
for (const entry of sitemapEntries) {
|
|
15046
15091
|
const locMatch = LOC_REGEX.exec(entry);
|
|
15047
15092
|
LOC_REGEX.lastIndex = 0;
|
|
15048
15093
|
if (locMatch?.[1]) {
|
|
15049
|
-
await parseSitemapRecursive(
|
|
15094
|
+
await parseSitemapRecursive(
|
|
15095
|
+
locMatch[1],
|
|
15096
|
+
urls,
|
|
15097
|
+
visited,
|
|
15098
|
+
depth + 1,
|
|
15099
|
+
/* isChild */
|
|
15100
|
+
true
|
|
15101
|
+
);
|
|
15050
15102
|
}
|
|
15051
15103
|
}
|
|
15052
15104
|
return;
|
|
@@ -15061,7 +15113,7 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
15061
15113
|
}
|
|
15062
15114
|
|
|
15063
15115
|
// src/gsc-inspect-sitemap.ts
|
|
15064
|
-
var
|
|
15116
|
+
var log4 = createLogger("InspectSitemap");
|
|
15065
15117
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
15066
15118
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15067
15119
|
db.update(runs).set({ status: "running", startedAt: now }).where(eq21(runs.id, runId)).run();
|
|
@@ -15094,9 +15146,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15094
15146
|
saveConfigPatch(opts.config);
|
|
15095
15147
|
}
|
|
15096
15148
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
15097
|
-
|
|
15149
|
+
log4.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
15098
15150
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
15099
|
-
|
|
15151
|
+
log4.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
15100
15152
|
if (urls.length === 0) {
|
|
15101
15153
|
throw new Error("No URLs found in sitemap");
|
|
15102
15154
|
}
|
|
@@ -15129,10 +15181,10 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15129
15181
|
createdAt: inspectedAt
|
|
15130
15182
|
}).run();
|
|
15131
15183
|
inspected++;
|
|
15132
|
-
|
|
15184
|
+
log4.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${urls.length}` });
|
|
15133
15185
|
} catch (err) {
|
|
15134
15186
|
errors++;
|
|
15135
|
-
|
|
15187
|
+
log4.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
15136
15188
|
}
|
|
15137
15189
|
if (inspected + errors < urls.length) {
|
|
15138
15190
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
@@ -15172,11 +15224,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15172
15224
|
}).run();
|
|
15173
15225
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
15174
15226
|
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
15175
|
-
|
|
15227
|
+
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
15176
15228
|
} catch (err) {
|
|
15177
15229
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15178
15230
|
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
15179
|
-
|
|
15231
|
+
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
15180
15232
|
throw err;
|
|
15181
15233
|
}
|
|
15182
15234
|
}
|
|
@@ -15184,7 +15236,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
15184
15236
|
// src/bing-inspect-sitemap.ts
|
|
15185
15237
|
import crypto22 from "crypto";
|
|
15186
15238
|
import { eq as eq22, desc as desc9 } from "drizzle-orm";
|
|
15187
|
-
var
|
|
15239
|
+
var log5 = createLogger("BingInspectSitemap");
|
|
15188
15240
|
function parseBingDate2(value) {
|
|
15189
15241
|
if (!value) return null;
|
|
15190
15242
|
const match = /\/Date\((-?\d+)[^)]*\)\//.exec(value);
|
|
@@ -15215,16 +15267,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15215
15267
|
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
15216
15268
|
}
|
|
15217
15269
|
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
15218
|
-
|
|
15270
|
+
log5.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
15219
15271
|
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
15220
|
-
|
|
15272
|
+
log5.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
15221
15273
|
if (sitemapUrls.length === 0) {
|
|
15222
15274
|
throw new Error("No URLs found in sitemap");
|
|
15223
15275
|
}
|
|
15224
15276
|
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq22(bingUrlInspections.projectId, projectId)).all();
|
|
15225
15277
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
15226
15278
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
15227
|
-
|
|
15279
|
+
log5.info("sitemap.diff", {
|
|
15228
15280
|
runId,
|
|
15229
15281
|
projectId,
|
|
15230
15282
|
sitemapTotal: sitemapUrls.length,
|
|
@@ -15239,9 +15291,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15239
15291
|
blockedUrls.add(issue.Url);
|
|
15240
15292
|
}
|
|
15241
15293
|
}
|
|
15242
|
-
|
|
15294
|
+
log5.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
15243
15295
|
} catch (err) {
|
|
15244
|
-
|
|
15296
|
+
log5.warn("crawl-issues.lookup-failed", {
|
|
15245
15297
|
runId,
|
|
15246
15298
|
projectId,
|
|
15247
15299
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -15285,7 +15337,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15285
15337
|
discoveryDate
|
|
15286
15338
|
}).run();
|
|
15287
15339
|
inspected++;
|
|
15288
|
-
|
|
15340
|
+
log5.info("inspect.url-done", {
|
|
15289
15341
|
runId,
|
|
15290
15342
|
projectId,
|
|
15291
15343
|
url: pageUrl,
|
|
@@ -15293,7 +15345,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15293
15345
|
});
|
|
15294
15346
|
} catch (err) {
|
|
15295
15347
|
errors++;
|
|
15296
|
-
|
|
15348
|
+
log5.error("inspect.url-failed", {
|
|
15297
15349
|
runId,
|
|
15298
15350
|
projectId,
|
|
15299
15351
|
url: pageUrl,
|
|
@@ -15348,7 +15400,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15348
15400
|
}).run();
|
|
15349
15401
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
15350
15402
|
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(runs.id, runId)).run();
|
|
15351
|
-
|
|
15403
|
+
log5.info("inspect.completed", {
|
|
15352
15404
|
runId,
|
|
15353
15405
|
projectId,
|
|
15354
15406
|
inspected,
|
|
@@ -15362,7 +15414,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15362
15414
|
} catch (err) {
|
|
15363
15415
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15364
15416
|
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(runs.id, runId)).run();
|
|
15365
|
-
|
|
15417
|
+
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
15366
15418
|
throw err;
|
|
15367
15419
|
}
|
|
15368
15420
|
}
|
|
@@ -15371,7 +15423,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
15371
15423
|
import crypto23 from "crypto";
|
|
15372
15424
|
import path11 from "path";
|
|
15373
15425
|
import { and as and11, eq as eq23, sql as sql8 } from "drizzle-orm";
|
|
15374
|
-
var
|
|
15426
|
+
var log6 = createLogger("CommonCrawlSync");
|
|
15375
15427
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
15376
15428
|
function defaultDeps() {
|
|
15377
15429
|
return {
|
|
@@ -15496,7 +15548,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15496
15548
|
updatedAt: finishedAt,
|
|
15497
15549
|
error: null
|
|
15498
15550
|
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15499
|
-
|
|
15551
|
+
log6.info("sync.completed", {
|
|
15500
15552
|
syncId,
|
|
15501
15553
|
release,
|
|
15502
15554
|
projectsProcessed: allProjects.length,
|
|
@@ -15508,7 +15560,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15508
15560
|
try {
|
|
15509
15561
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
15510
15562
|
} catch (err) {
|
|
15511
|
-
|
|
15563
|
+
log6.error("auto-extract.enqueue-failed", {
|
|
15512
15564
|
syncId,
|
|
15513
15565
|
release,
|
|
15514
15566
|
projectId: p.id,
|
|
@@ -15526,7 +15578,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
15526
15578
|
phaseDetail: null,
|
|
15527
15579
|
updatedAt: finishedAt
|
|
15528
15580
|
}).where(eq23(ccReleaseSyncs.id, syncId)).run();
|
|
15529
|
-
|
|
15581
|
+
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
15530
15582
|
throw err;
|
|
15531
15583
|
}
|
|
15532
15584
|
}
|
|
@@ -15562,7 +15614,7 @@ function computeSummary(rows) {
|
|
|
15562
15614
|
import crypto24 from "crypto";
|
|
15563
15615
|
import fs9 from "fs";
|
|
15564
15616
|
import { and as and12, desc as desc10, eq as eq24 } from "drizzle-orm";
|
|
15565
|
-
var
|
|
15617
|
+
var log7 = createLogger("BacklinkExtract");
|
|
15566
15618
|
function defaultDeps2() {
|
|
15567
15619
|
return {
|
|
15568
15620
|
queryBacklinks,
|
|
@@ -15648,7 +15700,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15648
15700
|
});
|
|
15649
15701
|
const finishedAt = deps.now().toISOString();
|
|
15650
15702
|
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq24(runs.id, runId)).run();
|
|
15651
|
-
|
|
15703
|
+
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
15652
15704
|
} catch (err) {
|
|
15653
15705
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15654
15706
|
const finishedAt = deps.now().toISOString();
|
|
@@ -15657,7 +15709,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
15657
15709
|
error: errorMsg,
|
|
15658
15710
|
finishedAt
|
|
15659
15711
|
}).where(eq24(runs.id, runId)).run();
|
|
15660
|
-
|
|
15712
|
+
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
15661
15713
|
throw err;
|
|
15662
15714
|
}
|
|
15663
15715
|
}
|
|
@@ -15730,7 +15782,7 @@ var ProviderRegistry = class {
|
|
|
15730
15782
|
// src/scheduler.ts
|
|
15731
15783
|
import cron from "node-cron";
|
|
15732
15784
|
import { eq as eq25 } from "drizzle-orm";
|
|
15733
|
-
var
|
|
15785
|
+
var log8 = createLogger("Scheduler");
|
|
15734
15786
|
var Scheduler = class {
|
|
15735
15787
|
db;
|
|
15736
15788
|
callbacks;
|
|
@@ -15746,11 +15798,11 @@ var Scheduler = class {
|
|
|
15746
15798
|
const missedRunAt = schedule.nextRunAt;
|
|
15747
15799
|
this.registerCronTask(schedule);
|
|
15748
15800
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
15749
|
-
|
|
15801
|
+
log8.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
15750
15802
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
15751
15803
|
}
|
|
15752
15804
|
}
|
|
15753
|
-
|
|
15805
|
+
log8.info("started", { scheduleCount: allSchedules.length });
|
|
15754
15806
|
}
|
|
15755
15807
|
/** Stop all cron tasks for graceful shutdown. */
|
|
15756
15808
|
stop() {
|
|
@@ -15782,12 +15834,12 @@ var Scheduler = class {
|
|
|
15782
15834
|
stopTask(projectId, task, verb) {
|
|
15783
15835
|
task.stop();
|
|
15784
15836
|
task.destroy();
|
|
15785
|
-
|
|
15837
|
+
log8.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
15786
15838
|
}
|
|
15787
15839
|
registerCronTask(schedule) {
|
|
15788
15840
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
15789
15841
|
if (!cron.validate(cronExpr)) {
|
|
15790
|
-
|
|
15842
|
+
log8.error("cron.invalid", { projectId, cronExpr });
|
|
15791
15843
|
return;
|
|
15792
15844
|
}
|
|
15793
15845
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -15801,14 +15853,14 @@ var Scheduler = class {
|
|
|
15801
15853
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15802
15854
|
}).where(eq25(schedules.id, scheduleId)).run();
|
|
15803
15855
|
const label = schedule.preset ?? cronExpr;
|
|
15804
|
-
|
|
15856
|
+
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
15805
15857
|
}
|
|
15806
15858
|
triggerRun(scheduleId, projectId) {
|
|
15807
15859
|
try {
|
|
15808
15860
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15809
15861
|
const currentSchedule = this.db.select().from(schedules).where(eq25(schedules.id, scheduleId)).get();
|
|
15810
15862
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
15811
|
-
|
|
15863
|
+
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
15812
15864
|
this.remove(projectId);
|
|
15813
15865
|
return;
|
|
15814
15866
|
}
|
|
@@ -15816,7 +15868,7 @@ var Scheduler = class {
|
|
|
15816
15868
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
15817
15869
|
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
15818
15870
|
if (!project) {
|
|
15819
|
-
|
|
15871
|
+
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
15820
15872
|
this.remove(projectId);
|
|
15821
15873
|
return;
|
|
15822
15874
|
}
|
|
@@ -15825,7 +15877,7 @@ var Scheduler = class {
|
|
|
15825
15877
|
if (project.defaultLocation) {
|
|
15826
15878
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
15827
15879
|
if (!loc) {
|
|
15828
|
-
|
|
15880
|
+
log8.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
15829
15881
|
return;
|
|
15830
15882
|
}
|
|
15831
15883
|
resolvedLocation = loc;
|
|
@@ -15839,7 +15891,7 @@ var Scheduler = class {
|
|
|
15839
15891
|
location: locationLabel
|
|
15840
15892
|
});
|
|
15841
15893
|
if (queueResult.conflict) {
|
|
15842
|
-
|
|
15894
|
+
log8.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
15843
15895
|
this.db.update(schedules).set({
|
|
15844
15896
|
nextRunAt,
|
|
15845
15897
|
updatedAt: now
|
|
@@ -15854,10 +15906,10 @@ var Scheduler = class {
|
|
|
15854
15906
|
}).where(eq25(schedules.id, currentSchedule.id)).run();
|
|
15855
15907
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
15856
15908
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
15857
|
-
|
|
15909
|
+
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
15858
15910
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
15859
15911
|
} catch (err) {
|
|
15860
|
-
|
|
15912
|
+
log8.error("trigger.error", { scheduleId, projectId, error: err instanceof Error ? err.message : String(err) });
|
|
15861
15913
|
}
|
|
15862
15914
|
}
|
|
15863
15915
|
};
|
|
@@ -15865,7 +15917,7 @@ var Scheduler = class {
|
|
|
15865
15917
|
// src/notifier.ts
|
|
15866
15918
|
import { eq as eq26, desc as desc11, and as and13, or as or2 } from "drizzle-orm";
|
|
15867
15919
|
import crypto25 from "crypto";
|
|
15868
|
-
var
|
|
15920
|
+
var log9 = createLogger("Notifier");
|
|
15869
15921
|
var Notifier = class {
|
|
15870
15922
|
db;
|
|
15871
15923
|
serverUrl;
|
|
@@ -15875,26 +15927,26 @@ var Notifier = class {
|
|
|
15875
15927
|
}
|
|
15876
15928
|
/** Called after a run completes (success, partial, or failed). */
|
|
15877
15929
|
async onRunCompleted(runId, projectId) {
|
|
15878
|
-
|
|
15930
|
+
log9.info("run.completed", { runId, projectId });
|
|
15879
15931
|
const notifs = this.db.select().from(notifications).where(eq26(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
15880
15932
|
if (notifs.length === 0) {
|
|
15881
|
-
|
|
15933
|
+
log9.info("notifications.none-enabled", { projectId });
|
|
15882
15934
|
return;
|
|
15883
15935
|
}
|
|
15884
|
-
|
|
15936
|
+
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
15885
15937
|
const run = this.db.select().from(runs).where(eq26(runs.id, runId)).get();
|
|
15886
15938
|
if (!run) {
|
|
15887
|
-
|
|
15939
|
+
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
15888
15940
|
return;
|
|
15889
15941
|
}
|
|
15890
15942
|
const project = this.db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
15891
15943
|
if (!project) {
|
|
15892
|
-
|
|
15944
|
+
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
15893
15945
|
return;
|
|
15894
15946
|
}
|
|
15895
15947
|
const transitions = this.computeTransitions(runId, projectId);
|
|
15896
15948
|
const events = [];
|
|
15897
|
-
|
|
15949
|
+
log9.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
15898
15950
|
if (run.status === "completed" || run.status === "partial") {
|
|
15899
15951
|
events.push("run.completed");
|
|
15900
15952
|
}
|
|
@@ -15910,7 +15962,7 @@ var Notifier = class {
|
|
|
15910
15962
|
if (!config.url) continue;
|
|
15911
15963
|
const subscribedEvents = config.events;
|
|
15912
15964
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
15913
|
-
|
|
15965
|
+
log9.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
15914
15966
|
if (matchingEvents.length === 0) continue;
|
|
15915
15967
|
for (const event of matchingEvents) {
|
|
15916
15968
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -16012,23 +16064,23 @@ var Notifier = class {
|
|
|
16012
16064
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
16013
16065
|
const targetCheck = await resolveWebhookTarget(url);
|
|
16014
16066
|
if (!targetCheck.ok) {
|
|
16015
|
-
|
|
16067
|
+
log9.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
16016
16068
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
16017
16069
|
return;
|
|
16018
16070
|
}
|
|
16019
|
-
|
|
16071
|
+
log9.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
16020
16072
|
const maxRetries = 3;
|
|
16021
16073
|
const delays = [1e3, 4e3, 16e3];
|
|
16022
16074
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
16023
16075
|
try {
|
|
16024
16076
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
16025
16077
|
if (response.status >= 200 && response.status < 300) {
|
|
16026
|
-
|
|
16078
|
+
log9.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
16027
16079
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
16028
16080
|
return;
|
|
16029
16081
|
}
|
|
16030
16082
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
16031
|
-
|
|
16083
|
+
log9.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
16032
16084
|
if (attempt === maxRetries - 1) {
|
|
16033
16085
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
16034
16086
|
}
|
|
@@ -16036,7 +16088,7 @@ var Notifier = class {
|
|
|
16036
16088
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
16037
16089
|
if (attempt === maxRetries - 1) {
|
|
16038
16090
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
16039
|
-
|
|
16091
|
+
log9.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
16040
16092
|
}
|
|
16041
16093
|
}
|
|
16042
16094
|
if (attempt < maxRetries - 1) {
|
|
@@ -16059,7 +16111,7 @@ var Notifier = class {
|
|
|
16059
16111
|
};
|
|
16060
16112
|
|
|
16061
16113
|
// src/run-coordinator.ts
|
|
16062
|
-
var
|
|
16114
|
+
var log10 = createLogger("RunCoordinator");
|
|
16063
16115
|
var RunCoordinator = class {
|
|
16064
16116
|
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
16065
16117
|
this.notifier = notifier;
|
|
@@ -16081,23 +16133,23 @@ var RunCoordinator = class {
|
|
|
16081
16133
|
try {
|
|
16082
16134
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
16083
16135
|
} catch (err) {
|
|
16084
|
-
|
|
16136
|
+
log10.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
16085
16137
|
}
|
|
16086
16138
|
}
|
|
16087
16139
|
}
|
|
16088
16140
|
} catch (err) {
|
|
16089
|
-
|
|
16141
|
+
log10.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
16090
16142
|
}
|
|
16091
16143
|
try {
|
|
16092
16144
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
16093
16145
|
} catch (err) {
|
|
16094
|
-
|
|
16146
|
+
log10.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
16095
16147
|
}
|
|
16096
16148
|
if (this.onAeroEvent) {
|
|
16097
16149
|
try {
|
|
16098
16150
|
await this.onAeroEvent({ runId, projectId, insightCount, criticalOrHigh });
|
|
16099
16151
|
} catch (err) {
|
|
16100
|
-
|
|
16152
|
+
log10.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
16101
16153
|
}
|
|
16102
16154
|
}
|
|
16103
16155
|
}
|
|
@@ -17041,7 +17093,7 @@ async function compactMessages(args) {
|
|
|
17041
17093
|
}
|
|
17042
17094
|
|
|
17043
17095
|
// src/agent/session-registry.ts
|
|
17044
|
-
var
|
|
17096
|
+
var log11 = createLogger("SessionRegistry");
|
|
17045
17097
|
var MAX_HYDRATE_NOTES = 20;
|
|
17046
17098
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
17047
17099
|
function escapeMemoryFragment(value) {
|
|
@@ -17272,13 +17324,13 @@ ${lines.join("\n")}
|
|
|
17272
17324
|
agent.state.messages = result.messages;
|
|
17273
17325
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
17274
17326
|
this.save(projectName);
|
|
17275
|
-
|
|
17327
|
+
log11.info("compaction.completed", {
|
|
17276
17328
|
projectName,
|
|
17277
17329
|
removedCount: result.removedCount,
|
|
17278
17330
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
17279
17331
|
});
|
|
17280
17332
|
} catch (err) {
|
|
17281
|
-
|
|
17333
|
+
log11.error("compaction.failed", {
|
|
17282
17334
|
projectName,
|
|
17283
17335
|
error: err instanceof Error ? err.message : String(err)
|
|
17284
17336
|
});
|
|
@@ -17375,7 +17427,7 @@ ${lines.join("\n")}
|
|
|
17375
17427
|
await agent.prompt(msgs);
|
|
17376
17428
|
this.save(projectName);
|
|
17377
17429
|
} catch (err) {
|
|
17378
|
-
|
|
17430
|
+
log11.error("drain.failed", {
|
|
17379
17431
|
projectName,
|
|
17380
17432
|
error: err instanceof Error ? err.message : String(err)
|
|
17381
17433
|
});
|
|
@@ -18353,7 +18405,7 @@ function formatAuditFactorScore(factor) {
|
|
|
18353
18405
|
}
|
|
18354
18406
|
|
|
18355
18407
|
// src/snapshot-service.ts
|
|
18356
|
-
var
|
|
18408
|
+
var log12 = createLogger("Snapshot");
|
|
18357
18409
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
18358
18410
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
18359
18411
|
var ProviderExecutionGate2 = class {
|
|
@@ -18496,7 +18548,7 @@ var SnapshotService = class {
|
|
|
18496
18548
|
return mapAuditReport(report);
|
|
18497
18549
|
} catch (err) {
|
|
18498
18550
|
const message = err instanceof Error ? err.message : String(err);
|
|
18499
|
-
|
|
18551
|
+
log12.warn("audit.failed", { homepageUrl, error: message });
|
|
18500
18552
|
return {
|
|
18501
18553
|
url: homepageUrl,
|
|
18502
18554
|
finalUrl: homepageUrl,
|
|
@@ -18526,7 +18578,7 @@ var SnapshotService = class {
|
|
|
18526
18578
|
phrases: parsedPhrases
|
|
18527
18579
|
};
|
|
18528
18580
|
} catch (err) {
|
|
18529
|
-
|
|
18581
|
+
log12.warn("profile.generation-failed", {
|
|
18530
18582
|
domain: ctx.domain,
|
|
18531
18583
|
provider: ctx.analysisProvider.adapter.name,
|
|
18532
18584
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -18668,7 +18720,7 @@ var SnapshotService = class {
|
|
|
18668
18720
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
18669
18721
|
};
|
|
18670
18722
|
} catch (err) {
|
|
18671
|
-
|
|
18723
|
+
log12.warn("response.analysis-failed", {
|
|
18672
18724
|
provider: ctx.analysisProvider.adapter.name,
|
|
18673
18725
|
error: err instanceof Error ? err.message : String(err)
|
|
18674
18726
|
});
|
|
@@ -18953,7 +19005,7 @@ function clipText(value, length) {
|
|
|
18953
19005
|
// src/server.ts
|
|
18954
19006
|
var _require2 = createRequire3(import.meta.url);
|
|
18955
19007
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
18956
|
-
var
|
|
19008
|
+
var log13 = createLogger("Server");
|
|
18957
19009
|
var DEFAULT_QUOTA = {
|
|
18958
19010
|
maxConcurrency: 2,
|
|
18959
19011
|
maxRequestsPerMinute: 10,
|
|
@@ -19040,7 +19092,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
19040
19092
|
}
|
|
19041
19093
|
if (migratedGoogle > 0) {
|
|
19042
19094
|
saveConfigPatch({ google: config.google });
|
|
19043
|
-
|
|
19095
|
+
log13.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
19044
19096
|
}
|
|
19045
19097
|
let migratedGa4 = 0;
|
|
19046
19098
|
for (const row of rows.ga4) {
|
|
@@ -19058,7 +19110,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
19058
19110
|
}
|
|
19059
19111
|
if (migratedGa4 > 0) {
|
|
19060
19112
|
saveConfigPatch({ ga4: config.ga4 });
|
|
19061
|
-
|
|
19113
|
+
log13.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
19062
19114
|
}
|
|
19063
19115
|
}
|
|
19064
19116
|
async function createServer(opts) {
|
|
@@ -19090,11 +19142,11 @@ async function createServer(opts) {
|
|
|
19090
19142
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
19091
19143
|
dropLegacyCredentialColumns(opts.db);
|
|
19092
19144
|
} catch (err) {
|
|
19093
|
-
|
|
19145
|
+
log13.warn("credentials.migration.failed", {
|
|
19094
19146
|
error: err instanceof Error ? err.message : String(err)
|
|
19095
19147
|
});
|
|
19096
19148
|
}
|
|
19097
|
-
|
|
19149
|
+
log13.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
19098
19150
|
const p = providers[k];
|
|
19099
19151
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
19100
19152
|
}) });
|
package/dist/cli.js
CHANGED
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ainyc/canonry",
|
|
3
|
-
"version": "2.5.
|
|
3
|
+
"version": "2.5.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Agent-first open-source AEO operating platform - track how answer engines cite your domain",
|
|
6
6
|
"license": "FSL-1.1-ALv2",
|
|
@@ -59,18 +59,18 @@
|
|
|
59
59
|
"tsx": "^4.19.0",
|
|
60
60
|
"@ainyc/canonry-api-routes": "0.0.0",
|
|
61
61
|
"@ainyc/canonry-config": "0.0.0",
|
|
62
|
-
"@ainyc/canonry-db": "0.0.0",
|
|
63
62
|
"@ainyc/canonry-contracts": "0.0.0",
|
|
63
|
+
"@ainyc/canonry-db": "0.0.0",
|
|
64
64
|
"@ainyc/canonry-intelligence": "0.0.0",
|
|
65
65
|
"@ainyc/canonry-integration-bing": "0.0.0",
|
|
66
66
|
"@ainyc/canonry-integration-commoncrawl": "0.0.0",
|
|
67
|
-
"@ainyc/canonry-integration-google": "0.0.0",
|
|
68
67
|
"@ainyc/canonry-integration-wordpress": "0.0.0",
|
|
69
|
-
"@ainyc/canonry-provider-claude": "0.0.0",
|
|
70
68
|
"@ainyc/canonry-provider-cdp": "0.0.0",
|
|
69
|
+
"@ainyc/canonry-provider-claude": "0.0.0",
|
|
70
|
+
"@ainyc/canonry-integration-google": "0.0.0",
|
|
71
71
|
"@ainyc/canonry-provider-local": "0.0.0",
|
|
72
|
-
"@ainyc/canonry-provider-openai": "0.0.0",
|
|
73
72
|
"@ainyc/canonry-provider-gemini": "0.0.0",
|
|
73
|
+
"@ainyc/canonry-provider-openai": "0.0.0",
|
|
74
74
|
"@ainyc/canonry-provider-perplexity": "0.0.0"
|
|
75
75
|
},
|
|
76
76
|
"scripts": {
|