@ainyc/canonry 4.14.0 → 4.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -223
- package/assets/assets/{index-D0EPNRDs.css → index-C1WW21tz.css} +1 -1
- package/assets/assets/index-Qq_oMI-C.js +302 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-5NYG5EC7.js → chunk-7SRKUAZO.js} +131 -2
- package/dist/{chunk-UQHWSCTE.js → chunk-IVNWS2YU.js} +473 -104
- package/dist/{chunk-7HBZCGRL.js → chunk-MI33SQL6.js} +27 -2
- package/dist/{chunk-6QTH5NS5.js → chunk-ONI3TX2A.js} +62 -0
- package/dist/cli.js +354 -119
- package/dist/index.d.ts +1 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-BCKXIKIL.js → intelligence-service-JYV3CO4H.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +7 -7
- package/assets/assets/index-B6Mi9Fd1.js +0 -302
|
@@ -3,8 +3,9 @@ import {
|
|
|
3
3
|
canonryMcpTools,
|
|
4
4
|
configExists,
|
|
5
5
|
loadConfig,
|
|
6
|
+
loadConfigRaw,
|
|
6
7
|
saveConfigPatch
|
|
7
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-7SRKUAZO.js";
|
|
8
9
|
import {
|
|
9
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
10
11
|
IntelligenceService,
|
|
@@ -65,7 +66,7 @@ import {
|
|
|
65
66
|
schedules,
|
|
66
67
|
trafficSources,
|
|
67
68
|
usageCounters
|
|
68
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-MI33SQL6.js";
|
|
69
70
|
import {
|
|
70
71
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
71
72
|
AGENT_PROVIDER_IDS,
|
|
@@ -81,6 +82,7 @@ import {
|
|
|
81
82
|
RunStatuses,
|
|
82
83
|
RunTriggers,
|
|
83
84
|
TrafficEventConfidences,
|
|
85
|
+
TrafficEventKinds,
|
|
84
86
|
TrafficEvidenceKinds,
|
|
85
87
|
TrafficSourceAuthModes,
|
|
86
88
|
TrafficSourceStatuses,
|
|
@@ -151,7 +153,7 @@ import {
|
|
|
151
153
|
visibilityStateFromAnswerMentioned,
|
|
152
154
|
windowCutoff,
|
|
153
155
|
wordpressEnvSchema
|
|
154
|
-
} from "./chunk-
|
|
156
|
+
} from "./chunk-ONI3TX2A.js";
|
|
155
157
|
|
|
156
158
|
// src/telemetry.ts
|
|
157
159
|
import crypto from "crypto";
|
|
@@ -163,6 +165,11 @@ var TELEMETRY_ENDPOINT = "https://ainyc.ai/api/telemetry";
|
|
|
163
165
|
var TIMEOUT_MS = 3e3;
|
|
164
166
|
var ANON_ID_ENV_VAR = "CANONRY_ANONYMOUS_ID";
|
|
165
167
|
var ANON_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
168
|
+
var SESSION_ID = crypto.randomUUID();
|
|
169
|
+
var CURRENT_SOURCE = "cli";
|
|
170
|
+
function setTelemetrySource(source) {
|
|
171
|
+
CURRENT_SOURCE = source;
|
|
172
|
+
}
|
|
166
173
|
function isTelemetryEnabled() {
|
|
167
174
|
if (process.env.CANONRY_TELEMETRY_DISABLED === "1") return false;
|
|
168
175
|
if (process.env.DO_NOT_TRACK === "1") return false;
|
|
@@ -250,19 +257,42 @@ function showFirstRunNotice() {
|
|
|
250
257
|
"\nCanonry collects anonymous telemetry to prioritize features.\nDisable any time: canonry telemetry disable\nLearn more: https://ainyc.ai/telemetry\n\n"
|
|
251
258
|
);
|
|
252
259
|
}
|
|
253
|
-
function
|
|
260
|
+
function detectAndTrackUpgrade() {
|
|
261
|
+
if (!isTelemetryEnabled()) return;
|
|
262
|
+
if (!configExists()) return;
|
|
263
|
+
let lastSeen;
|
|
264
|
+
try {
|
|
265
|
+
const raw = loadConfigRaw();
|
|
266
|
+
lastSeen = raw?.lastSeenVersion;
|
|
267
|
+
} catch {
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (lastSeen === VERSION) return;
|
|
271
|
+
try {
|
|
272
|
+
saveConfigPatch({ lastSeenVersion: VERSION });
|
|
273
|
+
} catch {
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
if (!lastSeen) return;
|
|
277
|
+
trackEvent("cli.upgraded", { fromVersion: lastSeen, toVersion: VERSION });
|
|
278
|
+
}
|
|
279
|
+
function trackEvent(event, properties, options) {
|
|
254
280
|
if (!isTelemetryEnabled()) return;
|
|
255
281
|
const anonymousId = getOrCreateAnonymousId();
|
|
256
282
|
if (!anonymousId) return;
|
|
257
283
|
const payload = {
|
|
258
284
|
anonymousId,
|
|
285
|
+
sessionId: SESSION_ID,
|
|
286
|
+
source: options?.source ?? CURRENT_SOURCE,
|
|
259
287
|
event,
|
|
260
288
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
261
289
|
version: VERSION,
|
|
262
290
|
nodeVersion: process.versions.node,
|
|
263
291
|
os: process.platform,
|
|
264
292
|
arch: process.arch,
|
|
265
|
-
|
|
293
|
+
...options?.sourceContext ? { sourceContext: options.sourceContext } : {},
|
|
294
|
+
...options?.errorCode ? { errorCode: options.errorCode } : {},
|
|
295
|
+
...properties ? { properties } : {}
|
|
266
296
|
};
|
|
267
297
|
const controller = new AbortController();
|
|
268
298
|
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
@@ -278,7 +308,7 @@ function trackEvent(event, properties) {
|
|
|
278
308
|
|
|
279
309
|
// src/server.ts
|
|
280
310
|
import { createRequire as createRequire3 } from "module";
|
|
281
|
-
import
|
|
311
|
+
import crypto31 from "crypto";
|
|
282
312
|
import fs12 from "fs";
|
|
283
313
|
import path14 from "path";
|
|
284
314
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
@@ -9632,8 +9662,66 @@ var routeCatalog = [
|
|
|
9632
9662
|
},
|
|
9633
9663
|
responses: {
|
|
9634
9664
|
200: { description: "Sync summary returned." },
|
|
9635
|
-
400: { description: "Invalid sync request
|
|
9636
|
-
404: { description: "Project or traffic source not found." }
|
|
9665
|
+
400: { description: "Invalid sync request or missing credentials." },
|
|
9666
|
+
404: { description: "Project or traffic source not found." },
|
|
9667
|
+
502: { description: "Upstream Cloud Run pull or auth-token resolution failed." }
|
|
9668
|
+
}
|
|
9669
|
+
},
|
|
9670
|
+
{
|
|
9671
|
+
method: "get",
|
|
9672
|
+
path: "/api/v1/projects/{name}/traffic/sources",
|
|
9673
|
+
summary: "List non-archived traffic sources for a project",
|
|
9674
|
+
tags: ["traffic"],
|
|
9675
|
+
parameters: [nameParameter],
|
|
9676
|
+
responses: {
|
|
9677
|
+
200: { description: "Source list returned." },
|
|
9678
|
+
404: { description: "Project not found." }
|
|
9679
|
+
}
|
|
9680
|
+
},
|
|
9681
|
+
{
|
|
9682
|
+
method: "get",
|
|
9683
|
+
path: "/api/v1/projects/{name}/traffic/status",
|
|
9684
|
+
summary: "List non-archived traffic sources with last-24h totals and the latest sync run for each",
|
|
9685
|
+
description: "Single-call composite for the `canonry traffic status` view: same shape as `GET /traffic/sources/{id}` but returned as `{ sources: TrafficSourceDetailDto[] }` for every non-archived source. Lets agents and the dashboard avoid an N+1 fan-out.",
|
|
9686
|
+
tags: ["traffic"],
|
|
9687
|
+
parameters: [nameParameter],
|
|
9688
|
+
responses: {
|
|
9689
|
+
200: { description: "Status returned." },
|
|
9690
|
+
404: { description: "Project not found." }
|
|
9691
|
+
}
|
|
9692
|
+
},
|
|
9693
|
+
{
|
|
9694
|
+
method: "get",
|
|
9695
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}",
|
|
9696
|
+
summary: "Get a single traffic source with last-24h totals and the latest sync run",
|
|
9697
|
+
tags: ["traffic"],
|
|
9698
|
+
parameters: [
|
|
9699
|
+
nameParameter,
|
|
9700
|
+
{ name: "id", in: "path", required: true, description: "Traffic source ID.", schema: stringSchema }
|
|
9701
|
+
],
|
|
9702
|
+
responses: {
|
|
9703
|
+
200: { description: "Source detail returned." },
|
|
9704
|
+
404: { description: "Project or source not found." }
|
|
9705
|
+
}
|
|
9706
|
+
},
|
|
9707
|
+
{
|
|
9708
|
+
method: "get",
|
|
9709
|
+
path: "/api/v1/projects/{name}/traffic/events",
|
|
9710
|
+
summary: "List rolled-up crawler and AI-referral hits within a window",
|
|
9711
|
+
description: "Returns hourly rollup rows from `crawler_events_hourly` and `ai_referral_events_hourly`. Defaults to the last 24h. Totals reflect the full window; the `events` array is capped by `limit` (default 500, max 5000).",
|
|
9712
|
+
tags: ["traffic"],
|
|
9713
|
+
parameters: [
|
|
9714
|
+
nameParameter,
|
|
9715
|
+
{ name: "since", in: "query", description: "ISO-8601 window start (defaults to 24h ago).", schema: stringSchema },
|
|
9716
|
+
{ name: "until", in: "query", description: "ISO-8601 window end (defaults to now).", schema: stringSchema },
|
|
9717
|
+
{ name: "kind", in: "query", description: 'Filter to "crawler", "ai-referral", or "all" (default).', schema: stringSchema },
|
|
9718
|
+
{ name: "limit", in: "query", description: "Max rows per kind in the events array (default 500, max 5000).", schema: stringSchema },
|
|
9719
|
+
{ name: "sourceId", in: "query", description: "Restrict to a single traffic source.", schema: stringSchema }
|
|
9720
|
+
],
|
|
9721
|
+
responses: {
|
|
9722
|
+
200: { description: "Events returned with windowed totals." },
|
|
9723
|
+
400: { description: "Invalid query parameters." },
|
|
9724
|
+
404: { description: "Project not found." }
|
|
9637
9725
|
}
|
|
9638
9726
|
}
|
|
9639
9727
|
];
|
|
@@ -15864,7 +15952,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
15864
15952
|
|
|
15865
15953
|
// ../api-routes/src/traffic.ts
|
|
15866
15954
|
import crypto20 from "crypto";
|
|
15867
|
-
import { eq as eq23, sql as sql7 } from "drizzle-orm";
|
|
15955
|
+
import { and as and12, desc as desc12, eq as eq23, gte, lte, sql as sql7 } from "drizzle-orm";
|
|
15868
15956
|
|
|
15869
15957
|
// ../integration-cloud-run/src/auth.ts
|
|
15870
15958
|
import crypto19 from "crypto";
|
|
@@ -16471,10 +16559,11 @@ function incrementBucket(map, key, fields) {
|
|
|
16471
16559
|
}
|
|
16472
16560
|
|
|
16473
16561
|
// ../api-routes/src/traffic.ts
|
|
16474
|
-
var DEFAULT_SYNC_WINDOW_MINUTES =
|
|
16562
|
+
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
16475
16563
|
var DEFAULT_PAGE_SIZE2 = 1e3;
|
|
16476
16564
|
var DEFAULT_MAX_PAGES2 = 5;
|
|
16477
16565
|
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
16566
|
+
var MAX_TRACKED_EVENT_IDS = 1e3;
|
|
16478
16567
|
function parseSourceConfig(row) {
|
|
16479
16568
|
return parseJsonColumn(row.configJson, {});
|
|
16480
16569
|
}
|
|
@@ -16635,17 +16724,24 @@ async function trafficRoutes(app, opts) {
|
|
|
16635
16724
|
kind: RunKinds["traffic-sync"],
|
|
16636
16725
|
status: RunStatuses.running,
|
|
16637
16726
|
trigger: RunTriggers.manual,
|
|
16727
|
+
sourceId: sourceRow.id,
|
|
16638
16728
|
startedAt,
|
|
16639
16729
|
createdAt: startedAt
|
|
16640
16730
|
}).run();
|
|
16731
|
+
const markFailed = (msg) => {
|
|
16732
|
+
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16733
|
+
app.db.transaction((tx) => {
|
|
16734
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq23(runs.id, runId)).run();
|
|
16735
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16736
|
+
});
|
|
16737
|
+
};
|
|
16641
16738
|
let accessToken;
|
|
16642
16739
|
try {
|
|
16643
16740
|
accessToken = await resolveAccessToken2(credential);
|
|
16644
16741
|
} catch (e) {
|
|
16645
16742
|
const msg = e instanceof Error ? e.message : String(e);
|
|
16646
|
-
|
|
16647
|
-
|
|
16648
|
-
throw validationError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
16743
|
+
markFailed(msg);
|
|
16744
|
+
throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
16649
16745
|
}
|
|
16650
16746
|
let allEvents = [];
|
|
16651
16747
|
try {
|
|
@@ -16661,11 +16757,23 @@ async function trafficRoutes(app, opts) {
|
|
|
16661
16757
|
allEvents = page.events;
|
|
16662
16758
|
} catch (e) {
|
|
16663
16759
|
const msg = e instanceof Error ? e.message : String(e);
|
|
16664
|
-
|
|
16665
|
-
|
|
16666
|
-
|
|
16667
|
-
|
|
16668
|
-
const
|
|
16760
|
+
markFailed(msg);
|
|
16761
|
+
throw providerError(`Cloud Run pull failed: ${msg}`);
|
|
16762
|
+
}
|
|
16763
|
+
const seenEventIds = new Set(parseJsonColumn(sourceRow.lastEventIds, []));
|
|
16764
|
+
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
16765
|
+
const newSorted = dedupedEvents.slice().sort((a, b) => a.observedAt < b.observedAt ? 1 : a.observedAt > b.observedAt ? -1 : 0).map((e) => e.eventId);
|
|
16766
|
+
const previousIds = parseJsonColumn(sourceRow.lastEventIds, []);
|
|
16767
|
+
const merged = [];
|
|
16768
|
+
const mergedSet = /* @__PURE__ */ new Set();
|
|
16769
|
+
for (const id of [...newSorted, ...previousIds]) {
|
|
16770
|
+
if (mergedSet.has(id)) continue;
|
|
16771
|
+
mergedSet.add(id);
|
|
16772
|
+
merged.push(id);
|
|
16773
|
+
if (merged.length >= MAX_TRACKED_EVENT_IDS) break;
|
|
16774
|
+
}
|
|
16775
|
+
const nextEventIds = merged;
|
|
16776
|
+
const report = buildTrafficProbeReport(dedupedEvents, { sampleLimit });
|
|
16669
16777
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16670
16778
|
let crawlerBucketRows = 0;
|
|
16671
16779
|
let aiReferralBucketRows = 0;
|
|
@@ -16771,6 +16879,7 @@ async function trafficRoutes(app, opts) {
|
|
|
16771
16879
|
status: TrafficSourceStatuses.connected,
|
|
16772
16880
|
lastSyncedAt: finishedAt,
|
|
16773
16881
|
lastError: null,
|
|
16882
|
+
lastEventIds: JSON.stringify(nextEventIds),
|
|
16774
16883
|
updatedAt: finishedAt
|
|
16775
16884
|
}).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16776
16885
|
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
@@ -16798,6 +16907,177 @@ async function trafficRoutes(app, opts) {
|
|
|
16798
16907
|
};
|
|
16799
16908
|
return response;
|
|
16800
16909
|
});
|
|
16910
|
+
function buildSourceDetail(projectId, row, since) {
|
|
16911
|
+
const crawlerTotals = app.db.select({ total: sql7`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
16912
|
+
and12(
|
|
16913
|
+
eq23(crawlerEventsHourly.sourceId, row.id),
|
|
16914
|
+
gte(crawlerEventsHourly.tsHour, since)
|
|
16915
|
+
)
|
|
16916
|
+
).get();
|
|
16917
|
+
const aiTotals = app.db.select({ total: sql7`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
16918
|
+
and12(
|
|
16919
|
+
eq23(aiReferralEventsHourly.sourceId, row.id),
|
|
16920
|
+
gte(aiReferralEventsHourly.tsHour, since)
|
|
16921
|
+
)
|
|
16922
|
+
).get();
|
|
16923
|
+
const sampleTotals = app.db.select({ total: sql7`COUNT(*)` }).from(rawEventSamples).where(
|
|
16924
|
+
and12(
|
|
16925
|
+
eq23(rawEventSamples.sourceId, row.id),
|
|
16926
|
+
gte(rawEventSamples.ts, since)
|
|
16927
|
+
)
|
|
16928
|
+
).get();
|
|
16929
|
+
const latestRun = app.db.select().from(runs).where(
|
|
16930
|
+
and12(
|
|
16931
|
+
eq23(runs.projectId, projectId),
|
|
16932
|
+
eq23(runs.kind, RunKinds["traffic-sync"]),
|
|
16933
|
+
eq23(runs.sourceId, row.id)
|
|
16934
|
+
)
|
|
16935
|
+
).orderBy(desc12(runs.startedAt)).limit(1).get();
|
|
16936
|
+
return {
|
|
16937
|
+
...rowToDto(row),
|
|
16938
|
+
totals24h: {
|
|
16939
|
+
crawlerHits: Number(crawlerTotals?.total ?? 0),
|
|
16940
|
+
aiReferralHits: Number(aiTotals?.total ?? 0),
|
|
16941
|
+
sampleCount: Number(sampleTotals?.total ?? 0)
|
|
16942
|
+
},
|
|
16943
|
+
latestRun: latestRun ? {
|
|
16944
|
+
runId: latestRun.id,
|
|
16945
|
+
status: latestRun.status,
|
|
16946
|
+
startedAt: latestRun.startedAt,
|
|
16947
|
+
finishedAt: latestRun.finishedAt ?? null,
|
|
16948
|
+
error: latestRun.error ?? null
|
|
16949
|
+
} : null
|
|
16950
|
+
};
|
|
16951
|
+
}
|
|
16952
|
+
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
16953
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16954
|
+
const rows = app.db.select().from(trafficSources).where(eq23(trafficSources.projectId, project.id)).orderBy(desc12(trafficSources.createdAt)).all();
|
|
16955
|
+
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
16956
|
+
const response = { sources };
|
|
16957
|
+
return response;
|
|
16958
|
+
});
|
|
16959
|
+
app.get("/projects/:name/traffic/status", async (request) => {
|
|
16960
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16961
|
+
const rows = app.db.select().from(trafficSources).where(eq23(trafficSources.projectId, project.id)).orderBy(desc12(trafficSources.createdAt)).all();
|
|
16962
|
+
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
16963
|
+
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
16964
|
+
const response = { sources };
|
|
16965
|
+
return response;
|
|
16966
|
+
});
|
|
16967
|
+
app.get(
|
|
16968
|
+
"/projects/:name/traffic/sources/:id",
|
|
16969
|
+
async (request) => {
|
|
16970
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16971
|
+
const row = app.db.select().from(trafficSources).where(eq23(trafficSources.id, request.params.id)).get();
|
|
16972
|
+
if (!row || row.projectId !== project.id) {
|
|
16973
|
+
throw notFound("Traffic source", request.params.id);
|
|
16974
|
+
}
|
|
16975
|
+
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
16976
|
+
return buildSourceDetail(project.id, row, since);
|
|
16977
|
+
}
|
|
16978
|
+
);
|
|
16979
|
+
app.get("/projects/:name/traffic/events", async (request) => {
|
|
16980
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16981
|
+
const now = /* @__PURE__ */ new Date();
|
|
16982
|
+
const defaultSince = new Date(now.getTime() - 24 * 60 * 6e4);
|
|
16983
|
+
const sinceParam = request.query?.since;
|
|
16984
|
+
const untilParam = request.query?.until;
|
|
16985
|
+
const since = sinceParam ? new Date(sinceParam) : defaultSince;
|
|
16986
|
+
const until = untilParam ? new Date(untilParam) : now;
|
|
16987
|
+
if (Number.isNaN(since.getTime())) {
|
|
16988
|
+
throw validationError('"since" must be an ISO-8601 timestamp');
|
|
16989
|
+
}
|
|
16990
|
+
if (Number.isNaN(until.getTime())) {
|
|
16991
|
+
throw validationError('"until" must be an ISO-8601 timestamp');
|
|
16992
|
+
}
|
|
16993
|
+
if (since.getTime() > until.getTime()) {
|
|
16994
|
+
throw validationError('"since" must be \u2264 "until"');
|
|
16995
|
+
}
|
|
16996
|
+
const kindParam = request.query?.kind;
|
|
16997
|
+
let kind = "all";
|
|
16998
|
+
if (kindParam !== void 0) {
|
|
16999
|
+
if (kindParam === "all" || kindParam === TrafficEventKinds.crawler || kindParam === TrafficEventKinds["ai-referral"]) {
|
|
17000
|
+
kind = kindParam;
|
|
17001
|
+
} else {
|
|
17002
|
+
throw validationError(`"kind" must be one of: all, ${TrafficEventKinds.crawler}, ${TrafficEventKinds["ai-referral"]}`);
|
|
17003
|
+
}
|
|
17004
|
+
}
|
|
17005
|
+
const limitParam = request.query?.limit;
|
|
17006
|
+
const requestedLimit = limitParam ? parseInt(limitParam, 10) : 500;
|
|
17007
|
+
if (!Number.isFinite(requestedLimit) || requestedLimit <= 0) {
|
|
17008
|
+
throw validationError('"limit" must be a positive integer');
|
|
17009
|
+
}
|
|
17010
|
+
const limit = Math.min(requestedLimit, 5e3);
|
|
17011
|
+
const sourceIdParam = request.query?.sourceId;
|
|
17012
|
+
const sinceIso = since.toISOString();
|
|
17013
|
+
const untilIso = until.toISOString();
|
|
17014
|
+
const events = [];
|
|
17015
|
+
let crawlerTotal = 0;
|
|
17016
|
+
let aiReferralTotal = 0;
|
|
17017
|
+
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
17018
|
+
const crawlerFilters = [
|
|
17019
|
+
eq23(crawlerEventsHourly.projectId, project.id),
|
|
17020
|
+
gte(crawlerEventsHourly.tsHour, sinceIso),
|
|
17021
|
+
lte(crawlerEventsHourly.tsHour, untilIso)
|
|
17022
|
+
];
|
|
17023
|
+
if (sourceIdParam) crawlerFilters.push(eq23(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
17024
|
+
const crawlerWhere = and12(...crawlerFilters);
|
|
17025
|
+
const total = app.db.select({ total: sql7`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(crawlerWhere).get();
|
|
17026
|
+
crawlerTotal = Number(total?.total ?? 0);
|
|
17027
|
+
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc12(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
17028
|
+
for (const r of rows) {
|
|
17029
|
+
events.push({
|
|
17030
|
+
kind: TrafficEventKinds.crawler,
|
|
17031
|
+
sourceId: r.sourceId,
|
|
17032
|
+
tsHour: r.tsHour,
|
|
17033
|
+
botId: r.botId,
|
|
17034
|
+
operator: r.operator,
|
|
17035
|
+
verificationStatus: r.verificationStatus,
|
|
17036
|
+
pathNormalized: r.pathNormalized,
|
|
17037
|
+
status: r.status,
|
|
17038
|
+
hits: r.hits
|
|
17039
|
+
});
|
|
17040
|
+
}
|
|
17041
|
+
}
|
|
17042
|
+
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
17043
|
+
const aiFilters = [
|
|
17044
|
+
eq23(aiReferralEventsHourly.projectId, project.id),
|
|
17045
|
+
gte(aiReferralEventsHourly.tsHour, sinceIso),
|
|
17046
|
+
lte(aiReferralEventsHourly.tsHour, untilIso)
|
|
17047
|
+
];
|
|
17048
|
+
if (sourceIdParam) aiFilters.push(eq23(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
17049
|
+
const aiWhere = and12(...aiFilters);
|
|
17050
|
+
const total = app.db.select({ total: sql7`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
17051
|
+
aiReferralTotal = Number(total?.total ?? 0);
|
|
17052
|
+
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc12(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
17053
|
+
for (const r of rows) {
|
|
17054
|
+
events.push({
|
|
17055
|
+
kind: TrafficEventKinds["ai-referral"],
|
|
17056
|
+
sourceId: r.sourceId,
|
|
17057
|
+
tsHour: r.tsHour,
|
|
17058
|
+
product: r.product,
|
|
17059
|
+
operator: r.operator,
|
|
17060
|
+
sourceDomain: r.sourceDomain,
|
|
17061
|
+
evidenceType: r.evidenceType,
|
|
17062
|
+
landingPathNormalized: r.landingPathNormalized,
|
|
17063
|
+
status: r.status,
|
|
17064
|
+
hits: r.sessionsOrHits
|
|
17065
|
+
});
|
|
17066
|
+
}
|
|
17067
|
+
}
|
|
17068
|
+
events.sort((a, b) => a.tsHour < b.tsHour ? 1 : a.tsHour > b.tsHour ? -1 : 0);
|
|
17069
|
+
const trimmed = events.slice(0, limit);
|
|
17070
|
+
const response = {
|
|
17071
|
+
windowStart: sinceIso,
|
|
17072
|
+
windowEnd: untilIso,
|
|
17073
|
+
totals: {
|
|
17074
|
+
crawlerHits: crawlerTotal,
|
|
17075
|
+
aiReferralHits: aiReferralTotal
|
|
17076
|
+
},
|
|
17077
|
+
events: trimmed
|
|
17078
|
+
};
|
|
17079
|
+
return response;
|
|
17080
|
+
});
|
|
16801
17081
|
}
|
|
16802
17082
|
|
|
16803
17083
|
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
@@ -20165,11 +20445,51 @@ function removeWordpressConnection(config, projectName) {
|
|
|
20165
20445
|
}
|
|
20166
20446
|
|
|
20167
20447
|
// src/job-runner.ts
|
|
20168
|
-
import
|
|
20448
|
+
import crypto22 from "crypto";
|
|
20169
20449
|
import fs7 from "fs";
|
|
20170
20450
|
import path9 from "path";
|
|
20171
20451
|
import os5 from "os";
|
|
20172
|
-
import { and as
|
|
20452
|
+
import { and as and13, eq as eq24, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
20453
|
+
|
|
20454
|
+
// src/run-telemetry.ts
|
|
20455
|
+
import crypto21 from "crypto";
|
|
20456
|
+
function extractRegistrableHost(input) {
|
|
20457
|
+
if (!input) return null;
|
|
20458
|
+
const trimmed = input.trim();
|
|
20459
|
+
if (!trimmed) return null;
|
|
20460
|
+
let host;
|
|
20461
|
+
try {
|
|
20462
|
+
const candidate = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
|
|
20463
|
+
host = new URL(candidate).hostname;
|
|
20464
|
+
} catch {
|
|
20465
|
+
return null;
|
|
20466
|
+
}
|
|
20467
|
+
host = host.toLowerCase();
|
|
20468
|
+
if (host.startsWith("www.")) host = host.slice(4);
|
|
20469
|
+
if (!host || !host.includes(".")) return null;
|
|
20470
|
+
return host;
|
|
20471
|
+
}
|
|
20472
|
+
function hashDomain(input) {
|
|
20473
|
+
const host = extractRegistrableHost(input);
|
|
20474
|
+
if (!host) return null;
|
|
20475
|
+
return crypto21.createHash("sha256").update(host).digest("hex");
|
|
20476
|
+
}
|
|
20477
|
+
function buildRunCompletedProps(input) {
|
|
20478
|
+
const totalMs = input.phases?.total_ms ?? Date.now() - input.startTime;
|
|
20479
|
+
const props = {
|
|
20480
|
+
status: input.status,
|
|
20481
|
+
providerCount: input.providerCount,
|
|
20482
|
+
providers: [...input.providers],
|
|
20483
|
+
queryCount: input.queryCount,
|
|
20484
|
+
durationMs: totalMs
|
|
20485
|
+
};
|
|
20486
|
+
if (input.trigger) props.trigger = input.trigger;
|
|
20487
|
+
const domainHash = hashDomain(input.canonicalDomain ?? null);
|
|
20488
|
+
if (domainHash) props.domainHash = domainHash;
|
|
20489
|
+
if (input.phases) props.phases = input.phases;
|
|
20490
|
+
if (input.location) props.location = input.location;
|
|
20491
|
+
return props;
|
|
20492
|
+
}
|
|
20173
20493
|
|
|
20174
20494
|
// src/citation-utils.ts
|
|
20175
20495
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -20481,20 +20801,26 @@ var JobRunner = class {
|
|
|
20481
20801
|
async executeRun(runId, projectId, providerOverride, locationOverride) {
|
|
20482
20802
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20483
20803
|
const startTime = Date.now();
|
|
20804
|
+
let providerCallStart;
|
|
20805
|
+
let providerCallEnd;
|
|
20484
20806
|
let runLocation;
|
|
20485
20807
|
let activeProviders = [];
|
|
20486
20808
|
let projectQueries = [];
|
|
20809
|
+
let runTrigger;
|
|
20810
|
+
let canonicalDomain;
|
|
20487
20811
|
const providerDispatchCounts = /* @__PURE__ */ new Map();
|
|
20488
20812
|
try {
|
|
20489
20813
|
const existingRun = this.getRunState(runId);
|
|
20490
20814
|
if (!existingRun) {
|
|
20491
20815
|
throw new Error(`Run ${runId} not found`);
|
|
20492
20816
|
}
|
|
20817
|
+
runTrigger = existingRun.trigger ?? void 0;
|
|
20493
20818
|
if (existingRun.status === "cancelled") {
|
|
20494
20819
|
this.handleCancelledRun(runId, projectId, startTime, {
|
|
20495
20820
|
providerCount: 0,
|
|
20496
20821
|
providers: [],
|
|
20497
|
-
queryCount: 0
|
|
20822
|
+
queryCount: 0,
|
|
20823
|
+
...runTrigger ? { trigger: runTrigger } : {}
|
|
20498
20824
|
});
|
|
20499
20825
|
return;
|
|
20500
20826
|
}
|
|
@@ -20502,13 +20828,14 @@ var JobRunner = class {
|
|
|
20502
20828
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
20503
20829
|
}
|
|
20504
20830
|
if (existingRun.status === "queued") {
|
|
20505
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20831
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and13(eq24(runs.id, runId), eq24(runs.status, "queued"))).run();
|
|
20506
20832
|
}
|
|
20507
20833
|
this.throwIfRunCancelled(runId);
|
|
20508
20834
|
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
20509
20835
|
if (!project) {
|
|
20510
20836
|
throw new Error(`Project ${projectId} not found`);
|
|
20511
20837
|
}
|
|
20838
|
+
canonicalDomain = project.canonicalDomain;
|
|
20512
20839
|
if (locationOverride === null) {
|
|
20513
20840
|
runLocation = void 0;
|
|
20514
20841
|
} else if (locationOverride) {
|
|
@@ -20536,7 +20863,9 @@ var JobRunner = class {
|
|
|
20536
20863
|
providerCount: activeProviders.length,
|
|
20537
20864
|
providers: activeProviders.map((provider) => provider.adapter.name),
|
|
20538
20865
|
queryCount: projectQueries.length,
|
|
20539
|
-
...runLocation ? { location: runLocation.label } : {}
|
|
20866
|
+
...runLocation ? { location: runLocation.label } : {},
|
|
20867
|
+
...runTrigger ? { trigger: runTrigger } : {},
|
|
20868
|
+
...canonicalDomain ? { canonicalDomain } : {}
|
|
20540
20869
|
};
|
|
20541
20870
|
const queriesPerProvider = projectQueries.length;
|
|
20542
20871
|
const todayPeriod = getCurrentUsageDay();
|
|
@@ -20602,7 +20931,7 @@ var JobRunner = class {
|
|
|
20602
20931
|
);
|
|
20603
20932
|
let screenshotRelPath = null;
|
|
20604
20933
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
20605
|
-
const snapshotId =
|
|
20934
|
+
const snapshotId = crypto22.randomUUID();
|
|
20606
20935
|
const screenshotDir = path9.join(os5.homedir(), ".canonry", "screenshots", runId);
|
|
20607
20936
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
20608
20937
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -20632,7 +20961,7 @@ var JobRunner = class {
|
|
|
20632
20961
|
}).run();
|
|
20633
20962
|
} else {
|
|
20634
20963
|
this.db.insert(querySnapshots).values({
|
|
20635
|
-
id:
|
|
20964
|
+
id: crypto22.randomUUID(),
|
|
20636
20965
|
runId,
|
|
20637
20966
|
queryId: q.id,
|
|
20638
20967
|
provider: providerName,
|
|
@@ -20668,6 +20997,7 @@ var JobRunner = class {
|
|
|
20668
20997
|
}
|
|
20669
20998
|
}
|
|
20670
20999
|
};
|
|
21000
|
+
providerCallStart = Date.now();
|
|
20671
21001
|
await runWithConcurrency(apiProviders, resolveProviderFanout(), async (registeredProvider) => {
|
|
20672
21002
|
await Promise.all(projectQueries.map(async (q) => {
|
|
20673
21003
|
await processQueryForProvider(registeredProvider, q);
|
|
@@ -20678,6 +21008,7 @@ var JobRunner = class {
|
|
|
20678
21008
|
await processQueryForProvider(registeredProvider, q);
|
|
20679
21009
|
}
|
|
20680
21010
|
}
|
|
21011
|
+
providerCallEnd = Date.now();
|
|
20681
21012
|
this.throwIfRunCancelled(runId);
|
|
20682
21013
|
const allFailed = totalSnapshotsInserted === 0 && providerErrors.size > 0;
|
|
20683
21014
|
const someFailed = providerErrors.size > 0;
|
|
@@ -20693,15 +21024,22 @@ var JobRunner = class {
|
|
|
20693
21024
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
20694
21025
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
20695
21026
|
const failureCode = providerErrors.size > 0 ? classifyProviderErrors(providerErrors) : void 0;
|
|
20696
|
-
|
|
20697
|
-
|
|
20698
|
-
|
|
20699
|
-
|
|
20700
|
-
|
|
20701
|
-
|
|
20702
|
-
|
|
20703
|
-
|
|
20704
|
-
|
|
21027
|
+
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
21028
|
+
trackEvent(
|
|
21029
|
+
"run.completed",
|
|
21030
|
+
buildRunCompletedProps({
|
|
21031
|
+
status: finalStatus,
|
|
21032
|
+
providerCount: executionContext.providerCount,
|
|
21033
|
+
providers: executionContext.providers,
|
|
21034
|
+
queryCount: executionContext.queryCount,
|
|
21035
|
+
startTime,
|
|
21036
|
+
trigger: executionContext.trigger,
|
|
21037
|
+
canonicalDomain: executionContext.canonicalDomain,
|
|
21038
|
+
phases,
|
|
21039
|
+
location: executionContext.location
|
|
21040
|
+
}),
|
|
21041
|
+
failureCode ? { errorCode: failureCode } : void 0
|
|
21042
|
+
);
|
|
20705
21043
|
this.incrementUsage(projectId, "runs", 1);
|
|
20706
21044
|
if (this.onRunCompleted) {
|
|
20707
21045
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
@@ -20713,7 +21051,9 @@ var JobRunner = class {
|
|
|
20713
21051
|
providerCount: activeProviders.length,
|
|
20714
21052
|
providers: activeProviders.map((provider) => provider.adapter.name),
|
|
20715
21053
|
queryCount: projectQueries.length,
|
|
20716
|
-
...runLocation ? { location: runLocation.label } : {}
|
|
21054
|
+
...runLocation ? { location: runLocation.label } : {},
|
|
21055
|
+
...runTrigger ? { trigger: runTrigger } : {},
|
|
21056
|
+
...canonicalDomain ? { canonicalDomain } : {}
|
|
20717
21057
|
};
|
|
20718
21058
|
if (err instanceof RunCancelledError || this.isRunCancelled(runId)) {
|
|
20719
21059
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
@@ -20728,25 +21068,36 @@ var JobRunner = class {
|
|
|
20728
21068
|
}).where(eq24(runs.id, runId)).run();
|
|
20729
21069
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
20730
21070
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
21071
|
+
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
20731
21072
|
if (abortReason) {
|
|
21073
|
+
const domainHash = hashDomain(executionContext.canonicalDomain ?? null);
|
|
20732
21074
|
trackEvent("run.aborted", {
|
|
20733
21075
|
reason: abortReason,
|
|
20734
21076
|
providerCount: executionContext.providerCount,
|
|
20735
21077
|
providers: executionContext.providers,
|
|
20736
21078
|
queryCount: executionContext.queryCount,
|
|
20737
21079
|
durationMs: Date.now() - startTime,
|
|
21080
|
+
...executionContext.trigger ? { trigger: executionContext.trigger } : {},
|
|
21081
|
+
...domainHash ? { domainHash } : {},
|
|
21082
|
+
...phases ? { phases } : {},
|
|
20738
21083
|
...executionContext.location ? { location: executionContext.location } : {}
|
|
20739
21084
|
});
|
|
20740
21085
|
} else {
|
|
20741
|
-
trackEvent(
|
|
20742
|
-
|
|
20743
|
-
|
|
20744
|
-
|
|
20745
|
-
|
|
20746
|
-
|
|
20747
|
-
|
|
20748
|
-
|
|
20749
|
-
|
|
21086
|
+
trackEvent(
|
|
21087
|
+
"run.completed",
|
|
21088
|
+
buildRunCompletedProps({
|
|
21089
|
+
status: "failed",
|
|
21090
|
+
providerCount: executionContext.providerCount,
|
|
21091
|
+
providers: executionContext.providers,
|
|
21092
|
+
queryCount: executionContext.queryCount,
|
|
21093
|
+
startTime,
|
|
21094
|
+
trigger: executionContext.trigger,
|
|
21095
|
+
canonicalDomain: executionContext.canonicalDomain,
|
|
21096
|
+
phases,
|
|
21097
|
+
location: executionContext.location
|
|
21098
|
+
}),
|
|
21099
|
+
{ errorCode: "UNKNOWN" }
|
|
21100
|
+
);
|
|
20750
21101
|
}
|
|
20751
21102
|
if (this.onRunCompleted) {
|
|
20752
21103
|
this.onRunCompleted(runId, projectId).catch((notifErr) => {
|
|
@@ -20759,7 +21110,7 @@ var JobRunner = class {
|
|
|
20759
21110
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20760
21111
|
const period = now.slice(0, 10);
|
|
20761
21112
|
this.db.insert(usageCounters).values({
|
|
20762
|
-
id:
|
|
21113
|
+
id: crypto22.randomUUID(),
|
|
20763
21114
|
scope,
|
|
20764
21115
|
period,
|
|
20765
21116
|
metric,
|
|
@@ -20780,7 +21131,8 @@ var JobRunner = class {
|
|
|
20780
21131
|
return this.db.select({
|
|
20781
21132
|
status: runs.status,
|
|
20782
21133
|
finishedAt: runs.finishedAt,
|
|
20783
|
-
error: runs.error
|
|
21134
|
+
error: runs.error,
|
|
21135
|
+
trigger: runs.trigger
|
|
20784
21136
|
}).from(runs).where(eq24(runs.id, runId)).get();
|
|
20785
21137
|
}
|
|
20786
21138
|
isRunCancelled(runId) {
|
|
@@ -20799,14 +21151,20 @@ var JobRunner = class {
|
|
|
20799
21151
|
error: currentRun.error ?? "Cancelled by user"
|
|
20800
21152
|
}).where(eq24(runs.id, runId)).run();
|
|
20801
21153
|
}
|
|
20802
|
-
trackEvent(
|
|
20803
|
-
|
|
20804
|
-
|
|
20805
|
-
|
|
20806
|
-
|
|
20807
|
-
|
|
20808
|
-
|
|
20809
|
-
|
|
21154
|
+
trackEvent(
|
|
21155
|
+
"run.completed",
|
|
21156
|
+
buildRunCompletedProps({
|
|
21157
|
+
status: "cancelled",
|
|
21158
|
+
providerCount: context.providerCount,
|
|
21159
|
+
providers: context.providers,
|
|
21160
|
+
queryCount: context.queryCount,
|
|
21161
|
+
startTime,
|
|
21162
|
+
trigger: context.trigger,
|
|
21163
|
+
canonicalDomain: context.canonicalDomain,
|
|
21164
|
+
location: context.location
|
|
21165
|
+
}),
|
|
21166
|
+
{ errorCode: "RUN_CANCELLED" }
|
|
21167
|
+
);
|
|
20810
21168
|
if (this.onRunCompleted) {
|
|
20811
21169
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
20812
21170
|
log.error("notification.callback-failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
@@ -20817,10 +21175,19 @@ var JobRunner = class {
|
|
|
20817
21175
|
function getCurrentUsageDay() {
|
|
20818
21176
|
return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
20819
21177
|
}
|
|
21178
|
+
function buildPhases(input) {
|
|
21179
|
+
const total_ms = Date.now() - input.startTime;
|
|
21180
|
+
if (input.providerCallStart === void 0) {
|
|
21181
|
+
return { setup_ms: total_ms, provider_call_ms: 0, total_ms };
|
|
21182
|
+
}
|
|
21183
|
+
const setup_ms = input.providerCallStart - input.startTime;
|
|
21184
|
+
const provider_call_ms = (input.providerCallEnd ?? Date.now()) - input.providerCallStart;
|
|
21185
|
+
return { setup_ms, provider_call_ms, total_ms };
|
|
21186
|
+
}
|
|
20820
21187
|
|
|
20821
21188
|
// src/gsc-sync.ts
|
|
20822
|
-
import
|
|
20823
|
-
import { eq as eq25, and as
|
|
21189
|
+
import crypto23 from "crypto";
|
|
21190
|
+
import { eq as eq25, and as and14, sql as sql9 } from "drizzle-orm";
|
|
20824
21191
|
var log2 = createLogger("GscSync");
|
|
20825
21192
|
function formatDate3(d) {
|
|
20826
21193
|
return d.toISOString().split("T")[0];
|
|
@@ -20872,7 +21239,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
20872
21239
|
});
|
|
20873
21240
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
20874
21241
|
db.delete(gscSearchData).where(
|
|
20875
|
-
|
|
21242
|
+
and14(
|
|
20876
21243
|
eq25(gscSearchData.projectId, projectId),
|
|
20877
21244
|
sql9`${gscSearchData.date} >= ${startDate}`,
|
|
20878
21245
|
sql9`${gscSearchData.date} <= ${endDate}`
|
|
@@ -20885,7 +21252,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
20885
21252
|
for (const row of batch) {
|
|
20886
21253
|
const [query, page, country, device, date] = row.keys;
|
|
20887
21254
|
db.insert(gscSearchData).values({
|
|
20888
|
-
id:
|
|
21255
|
+
id: crypto23.randomUUID(),
|
|
20889
21256
|
projectId,
|
|
20890
21257
|
syncRunId: runId,
|
|
20891
21258
|
date: date ?? "",
|
|
@@ -20919,7 +21286,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
20919
21286
|
const rich = ir.richResultsResult;
|
|
20920
21287
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
20921
21288
|
db.insert(gscUrlInspections).values({
|
|
20922
|
-
id:
|
|
21289
|
+
id: crypto23.randomUUID(),
|
|
20923
21290
|
projectId,
|
|
20924
21291
|
syncRunId: runId,
|
|
20925
21292
|
url: pageUrl,
|
|
@@ -20961,9 +21328,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
20961
21328
|
}
|
|
20962
21329
|
}
|
|
20963
21330
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
20964
|
-
db.delete(gscCoverageSnapshots).where(
|
|
21331
|
+
db.delete(gscCoverageSnapshots).where(and14(eq25(gscCoverageSnapshots.projectId, projectId), eq25(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
20965
21332
|
db.insert(gscCoverageSnapshots).values({
|
|
20966
|
-
id:
|
|
21333
|
+
id: crypto23.randomUUID(),
|
|
20967
21334
|
projectId,
|
|
20968
21335
|
syncRunId: runId,
|
|
20969
21336
|
date: snapshotDate,
|
|
@@ -20983,8 +21350,8 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
20983
21350
|
}
|
|
20984
21351
|
|
|
20985
21352
|
// src/gsc-inspect-sitemap.ts
|
|
20986
|
-
import
|
|
20987
|
-
import { eq as eq26, and as
|
|
21353
|
+
import crypto24 from "crypto";
|
|
21354
|
+
import { eq as eq26, and as and15 } from "drizzle-orm";
|
|
20988
21355
|
|
|
20989
21356
|
// src/sitemap-parser.ts
|
|
20990
21357
|
var log3 = createLogger("SitemapParser");
|
|
@@ -21152,7 +21519,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21152
21519
|
const rich = ir.richResultsResult;
|
|
21153
21520
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
21154
21521
|
db.insert(gscUrlInspections).values({
|
|
21155
|
-
id:
|
|
21522
|
+
id: crypto24.randomUUID(),
|
|
21156
21523
|
projectId,
|
|
21157
21524
|
syncRunId: runId,
|
|
21158
21525
|
url: pageUrl,
|
|
@@ -21200,9 +21567,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21200
21567
|
}
|
|
21201
21568
|
}
|
|
21202
21569
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
21203
|
-
db.delete(gscCoverageSnapshots).where(
|
|
21570
|
+
db.delete(gscCoverageSnapshots).where(and15(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
21204
21571
|
db.insert(gscCoverageSnapshots).values({
|
|
21205
|
-
id:
|
|
21572
|
+
id: crypto24.randomUUID(),
|
|
21206
21573
|
projectId,
|
|
21207
21574
|
syncRunId: runId,
|
|
21208
21575
|
date: snapshotDate,
|
|
@@ -21223,8 +21590,8 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
21223
21590
|
}
|
|
21224
21591
|
|
|
21225
21592
|
// src/bing-inspect-sitemap.ts
|
|
21226
|
-
import
|
|
21227
|
-
import { eq as eq27, desc as
|
|
21593
|
+
import crypto25 from "crypto";
|
|
21594
|
+
import { eq as eq27, desc as desc13 } from "drizzle-orm";
|
|
21228
21595
|
var log5 = createLogger("BingInspectSitemap");
|
|
21229
21596
|
function parseBingDate2(value) {
|
|
21230
21597
|
if (!value) return null;
|
|
@@ -21311,7 +21678,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21311
21678
|
derivedInIndex = false;
|
|
21312
21679
|
}
|
|
21313
21680
|
db.insert(bingUrlInspections).values({
|
|
21314
|
-
id:
|
|
21681
|
+
id: crypto25.randomUUID(),
|
|
21315
21682
|
projectId,
|
|
21316
21683
|
url: pageUrl,
|
|
21317
21684
|
httpCode,
|
|
@@ -21345,7 +21712,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21345
21712
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
21346
21713
|
}
|
|
21347
21714
|
}
|
|
21348
|
-
const allInspections = db.select().from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).orderBy(
|
|
21715
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).orderBy(desc13(bingUrlInspections.inspectedAt)).all();
|
|
21349
21716
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
21350
21717
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
21351
21718
|
for (const row of allInspections) {
|
|
@@ -21369,7 +21736,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21369
21736
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
21370
21737
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
21371
21738
|
db.insert(bingCoverageSnapshots).values({
|
|
21372
|
-
id:
|
|
21739
|
+
id: crypto25.randomUUID(),
|
|
21373
21740
|
projectId,
|
|
21374
21741
|
syncRunId: runId,
|
|
21375
21742
|
date: snapshotDate,
|
|
@@ -21409,9 +21776,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
21409
21776
|
}
|
|
21410
21777
|
|
|
21411
21778
|
// src/commoncrawl-sync.ts
|
|
21412
|
-
import
|
|
21779
|
+
import crypto26 from "crypto";
|
|
21413
21780
|
import path10 from "path";
|
|
21414
|
-
import { and as
|
|
21781
|
+
import { and as and16, eq as eq28, sql as sql10 } from "drizzle-orm";
|
|
21415
21782
|
var log6 = createLogger("CommonCrawlSync");
|
|
21416
21783
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
21417
21784
|
function defaultDeps() {
|
|
@@ -21484,7 +21851,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21484
21851
|
if (!projectIds) continue;
|
|
21485
21852
|
for (const projectId of projectIds) {
|
|
21486
21853
|
expanded.push({
|
|
21487
|
-
id:
|
|
21854
|
+
id: crypto26.randomUUID(),
|
|
21488
21855
|
projectId,
|
|
21489
21856
|
releaseSyncId: syncId,
|
|
21490
21857
|
release,
|
|
@@ -21504,7 +21871,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
21504
21871
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
21505
21872
|
const summary = computeSummary(projectRows);
|
|
21506
21873
|
tx.insert(backlinkSummaries).values({
|
|
21507
|
-
id:
|
|
21874
|
+
id: crypto26.randomUUID(),
|
|
21508
21875
|
projectId: p.id,
|
|
21509
21876
|
releaseSyncId: syncId,
|
|
21510
21877
|
release,
|
|
@@ -21600,9 +21967,9 @@ function computeSummary(rows) {
|
|
|
21600
21967
|
}
|
|
21601
21968
|
|
|
21602
21969
|
// src/backlink-extract.ts
|
|
21603
|
-
import
|
|
21970
|
+
import crypto27 from "crypto";
|
|
21604
21971
|
import fs8 from "fs";
|
|
21605
|
-
import { and as
|
|
21972
|
+
import { and as and17, desc as desc14, eq as eq29 } from "drizzle-orm";
|
|
21606
21973
|
var log7 = createLogger("BacklinkExtract");
|
|
21607
21974
|
function defaultDeps2() {
|
|
21608
21975
|
return {
|
|
@@ -21620,7 +21987,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
21620
21987
|
if (!project) {
|
|
21621
21988
|
throw new Error(`Project not found: ${projectId}`);
|
|
21622
21989
|
}
|
|
21623
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(
|
|
21990
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc14(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
21624
21991
|
if (!sync) {
|
|
21625
21992
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
21626
21993
|
}
|
|
@@ -21648,11 +22015,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
21648
22015
|
const targetDomain = project.canonicalDomain;
|
|
21649
22016
|
db.transaction((tx) => {
|
|
21650
22017
|
tx.delete(backlinkDomains).where(
|
|
21651
|
-
|
|
22018
|
+
and17(eq29(backlinkDomains.projectId, projectId), eq29(backlinkDomains.release, release))
|
|
21652
22019
|
).run();
|
|
21653
22020
|
if (rows.length > 0) {
|
|
21654
22021
|
const values = rows.map((r) => ({
|
|
21655
|
-
id:
|
|
22022
|
+
id: crypto27.randomUUID(),
|
|
21656
22023
|
projectId,
|
|
21657
22024
|
releaseSyncId: syncId,
|
|
21658
22025
|
release,
|
|
@@ -21665,7 +22032,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
21665
22032
|
}
|
|
21666
22033
|
const summary = computeSummary2(rows);
|
|
21667
22034
|
tx.insert(backlinkSummaries).values({
|
|
21668
|
-
id:
|
|
22035
|
+
id: crypto27.randomUUID(),
|
|
21669
22036
|
projectId,
|
|
21670
22037
|
releaseSyncId: syncId,
|
|
21671
22038
|
release,
|
|
@@ -21904,8 +22271,8 @@ var Scheduler = class {
|
|
|
21904
22271
|
};
|
|
21905
22272
|
|
|
21906
22273
|
// src/notifier.ts
|
|
21907
|
-
import { eq as eq31, desc as
|
|
21908
|
-
import
|
|
22274
|
+
import { eq as eq31, desc as desc15, and as and18, or as or4 } from "drizzle-orm";
|
|
22275
|
+
import crypto28 from "crypto";
|
|
21909
22276
|
var log9 = createLogger("Notifier");
|
|
21910
22277
|
var Notifier = class {
|
|
21911
22278
|
db;
|
|
@@ -22010,11 +22377,11 @@ var Notifier = class {
|
|
|
22010
22377
|
}
|
|
22011
22378
|
computeTransitions(runId, projectId) {
|
|
22012
22379
|
const recentRuns = this.db.select().from(runs).where(
|
|
22013
|
-
|
|
22380
|
+
and18(
|
|
22014
22381
|
eq31(runs.projectId, projectId),
|
|
22015
22382
|
or4(eq31(runs.status, "completed"), eq31(runs.status, "partial"))
|
|
22016
22383
|
)
|
|
22017
|
-
).orderBy(
|
|
22384
|
+
).orderBy(desc15(runs.createdAt)).limit(2).all();
|
|
22018
22385
|
if (recentRuns.length < 2) return [];
|
|
22019
22386
|
const currentRunId = recentRuns[0].id;
|
|
22020
22387
|
const previousRunId = recentRuns[1].id;
|
|
@@ -22087,7 +22454,7 @@ var Notifier = class {
|
|
|
22087
22454
|
}
|
|
22088
22455
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
22089
22456
|
this.db.insert(auditLog).values({
|
|
22090
|
-
id:
|
|
22457
|
+
id: crypto28.randomUUID(),
|
|
22091
22458
|
projectId,
|
|
22092
22459
|
actor: "scheduler",
|
|
22093
22460
|
action: `notification.${status}`,
|
|
@@ -22145,7 +22512,7 @@ var RunCoordinator = class {
|
|
|
22145
22512
|
};
|
|
22146
22513
|
|
|
22147
22514
|
// src/agent/session-registry.ts
|
|
22148
|
-
import
|
|
22515
|
+
import crypto30 from "crypto";
|
|
22149
22516
|
import { eq as eq33 } from "drizzle-orm";
|
|
22150
22517
|
|
|
22151
22518
|
// src/agent/session.ts
|
|
@@ -22495,8 +22862,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
22495
22862
|
}
|
|
22496
22863
|
|
|
22497
22864
|
// src/agent/memory-store.ts
|
|
22498
|
-
import
|
|
22499
|
-
import { and as
|
|
22865
|
+
import crypto29 from "crypto";
|
|
22866
|
+
import { and as and19, desc as desc16, eq as eq32, like as like2, sql as sql11 } from "drizzle-orm";
|
|
22500
22867
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
22501
22868
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
22502
22869
|
function rowToDto2(row) {
|
|
@@ -22510,7 +22877,7 @@ function rowToDto2(row) {
|
|
|
22510
22877
|
};
|
|
22511
22878
|
}
|
|
22512
22879
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
22513
|
-
const query = db.select().from(agentMemory).where(eq32(agentMemory.projectId, projectId)).orderBy(
|
|
22880
|
+
const query = db.select().from(agentMemory).where(eq32(agentMemory.projectId, projectId)).orderBy(desc16(agentMemory.updatedAt));
|
|
22514
22881
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
22515
22882
|
return rows.map(rowToDto2);
|
|
22516
22883
|
}
|
|
@@ -22524,7 +22891,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
22524
22891
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
22525
22892
|
}
|
|
22526
22893
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22527
|
-
const id =
|
|
22894
|
+
const id = crypto29.randomUUID();
|
|
22528
22895
|
db.insert(agentMemory).values({
|
|
22529
22896
|
id,
|
|
22530
22897
|
projectId: args.projectId,
|
|
@@ -22541,12 +22908,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
22541
22908
|
updatedAt: now
|
|
22542
22909
|
}
|
|
22543
22910
|
}).run();
|
|
22544
|
-
const row = db.select().from(agentMemory).where(
|
|
22911
|
+
const row = db.select().from(agentMemory).where(and19(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, args.key))).get();
|
|
22545
22912
|
if (!row) throw new Error("memory upsert produced no row");
|
|
22546
22913
|
return rowToDto2(row);
|
|
22547
22914
|
}
|
|
22548
22915
|
function deleteMemoryEntry(db, projectId, key) {
|
|
22549
|
-
const result = db.delete(agentMemory).where(
|
|
22916
|
+
const result = db.delete(agentMemory).where(and19(eq32(agentMemory.projectId, projectId), eq32(agentMemory.key, key))).run();
|
|
22550
22917
|
const changes = result.changes ?? 0;
|
|
22551
22918
|
return changes > 0;
|
|
22552
22919
|
}
|
|
@@ -22561,7 +22928,7 @@ function writeCompactionNote(db, args) {
|
|
|
22561
22928
|
}
|
|
22562
22929
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22563
22930
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
22564
|
-
const id =
|
|
22931
|
+
const id = crypto29.randomUUID();
|
|
22565
22932
|
let inserted;
|
|
22566
22933
|
db.transaction((tx) => {
|
|
22567
22934
|
tx.insert(agentMemory).values({
|
|
@@ -22575,16 +22942,16 @@ function writeCompactionNote(db, args) {
|
|
|
22575
22942
|
}).run();
|
|
22576
22943
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
22577
22944
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
22578
|
-
|
|
22945
|
+
and19(
|
|
22579
22946
|
eq32(agentMemory.projectId, args.projectId),
|
|
22580
22947
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
22581
22948
|
)
|
|
22582
|
-
).orderBy(
|
|
22949
|
+
).orderBy(desc16(agentMemory.updatedAt)).all();
|
|
22583
22950
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
22584
22951
|
if (stale.length > 0) {
|
|
22585
22952
|
tx.delete(agentMemory).where(sql11`${agentMemory.id} IN (${sql11.join(stale.map((s) => sql11`${s}`), sql11`, `)})`).run();
|
|
22586
22953
|
}
|
|
22587
|
-
const row = tx.select().from(agentMemory).where(
|
|
22954
|
+
const row = tx.select().from(agentMemory).where(and19(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, key))).get();
|
|
22588
22955
|
if (row) inserted = rowToDto2(row);
|
|
22589
22956
|
});
|
|
22590
22957
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -23152,7 +23519,7 @@ ${lines.join("\n")}
|
|
|
23152
23519
|
insertRow(params) {
|
|
23153
23520
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23154
23521
|
this.opts.db.insert(agentSessions).values({
|
|
23155
|
-
id:
|
|
23522
|
+
id: crypto30.randomUUID(),
|
|
23156
23523
|
projectId: params.projectId,
|
|
23157
23524
|
systemPrompt: params.systemPrompt,
|
|
23158
23525
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -24069,7 +24436,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
24069
24436
|
};
|
|
24070
24437
|
}
|
|
24071
24438
|
function hashApiKey(key) {
|
|
24072
|
-
return
|
|
24439
|
+
return crypto31.createHash("sha256").update(key).digest("hex");
|
|
24073
24440
|
}
|
|
24074
24441
|
function parseCookies2(header) {
|
|
24075
24442
|
if (!header) return {};
|
|
@@ -24336,7 +24703,7 @@ async function createServer(opts) {
|
|
|
24336
24703
|
return removed;
|
|
24337
24704
|
}
|
|
24338
24705
|
};
|
|
24339
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
24706
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto31.randomBytes(32).toString("hex");
|
|
24340
24707
|
const googleConnectionStore = {
|
|
24341
24708
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
24342
24709
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -24386,7 +24753,7 @@ async function createServer(opts) {
|
|
|
24386
24753
|
if (!existing) {
|
|
24387
24754
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
24388
24755
|
opts.db.insert(apiKeys).values({
|
|
24389
|
-
id: `key_${
|
|
24756
|
+
id: `key_${crypto31.randomBytes(8).toString("hex")}`,
|
|
24390
24757
|
name: "default",
|
|
24391
24758
|
keyHash,
|
|
24392
24759
|
keyPrefix: prefix,
|
|
@@ -24410,7 +24777,7 @@ async function createServer(opts) {
|
|
|
24410
24777
|
};
|
|
24411
24778
|
const createSession = (apiKeyId) => {
|
|
24412
24779
|
pruneExpiredSessions();
|
|
24413
|
-
const sessionId =
|
|
24780
|
+
const sessionId = crypto31.randomBytes(32).toString("hex");
|
|
24414
24781
|
sessions.set(sessionId, {
|
|
24415
24782
|
apiKeyId,
|
|
24416
24783
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -24606,7 +24973,7 @@ async function createServer(opts) {
|
|
|
24606
24973
|
deps: {
|
|
24607
24974
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
24608
24975
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
24609
|
-
const runId =
|
|
24976
|
+
const runId = crypto31.randomUUID();
|
|
24610
24977
|
opts.db.insert(runs).values({
|
|
24611
24978
|
id: runId,
|
|
24612
24979
|
projectId,
|
|
@@ -24742,7 +25109,7 @@ async function createServer(opts) {
|
|
|
24742
25109
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
24743
25110
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
24744
25111
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
24745
|
-
id:
|
|
25112
|
+
id: crypto31.randomUUID(),
|
|
24746
25113
|
projectId,
|
|
24747
25114
|
actor: "api",
|
|
24748
25115
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -24985,10 +25352,12 @@ function parseQueryResponse(raw, count) {
|
|
|
24985
25352
|
}
|
|
24986
25353
|
|
|
24987
25354
|
export {
|
|
25355
|
+
setTelemetrySource,
|
|
24988
25356
|
isTelemetryEnabled,
|
|
24989
25357
|
getOrCreateAnonymousId,
|
|
24990
25358
|
isFirstRun,
|
|
24991
25359
|
showFirstRunNotice,
|
|
25360
|
+
detectAndTrackUpgrade,
|
|
24992
25361
|
trackEvent,
|
|
24993
25362
|
reparseStoredResult2 as reparseStoredResult,
|
|
24994
25363
|
reparseStoredResult3 as reparseStoredResult2,
|