@ainyc/canonry 4.19.1 → 4.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{index-CVqSCXSn.js → index-CChQIKw5.js} +110 -110
- package/assets/index.html +1 -1
- package/dist/{chunk-OHPZXTFC.js → chunk-3UGJUNQX.js} +301 -6
- package/dist/{chunk-SBZTDECX.js → chunk-EY63PENL.js} +15 -1
- package/dist/{chunk-BN2VQDZ2.js → chunk-GVQYROIK.js} +1 -1
- package/dist/{chunk-P3SFTXHG.js → chunk-VFKGHXVJ.js} +24 -1
- package/dist/cli.js +104 -6
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-6CX5HH27.js → intelligence-service-5COCQKXG.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +8 -8
package/assets/index.html
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
<link rel="icon" type="image/png" sizes="32x32" href="./favicon-32.png" />
|
|
13
13
|
<link rel="apple-touch-icon" href="./apple-touch-icon.png" />
|
|
14
14
|
<title>Canonry</title>
|
|
15
|
-
<script type="module" crossorigin src="./assets/index-
|
|
15
|
+
<script type="module" crossorigin src="./assets/index-CChQIKw5.js"></script>
|
|
16
16
|
<link rel="stylesheet" crossorigin href="./assets/index-QBgWzl2L.css">
|
|
17
17
|
</head>
|
|
18
18
|
<body>
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-VFKGHXVJ.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
schedules,
|
|
67
67
|
trafficSources,
|
|
68
68
|
usageCounters
|
|
69
|
-
} from "./chunk-
|
|
69
|
+
} from "./chunk-GVQYROIK.js";
|
|
70
70
|
import {
|
|
71
71
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
72
72
|
AGENT_PROVIDER_IDS,
|
|
@@ -159,7 +159,7 @@ import {
|
|
|
159
159
|
visibilityStateFromAnswerMentioned,
|
|
160
160
|
windowCutoff,
|
|
161
161
|
wordpressEnvSchema
|
|
162
|
-
} from "./chunk-
|
|
162
|
+
} from "./chunk-EY63PENL.js";
|
|
163
163
|
|
|
164
164
|
// src/telemetry.ts
|
|
165
165
|
import crypto from "crypto";
|
|
@@ -10037,6 +10037,35 @@ var routeCatalog = [
|
|
|
10037
10037
|
502: { description: "Upstream Cloud Run pull or auth-token resolution failed." }
|
|
10038
10038
|
}
|
|
10039
10039
|
},
|
|
10040
|
+
{
|
|
10041
|
+
method: "post",
|
|
10042
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/backfill",
|
|
10043
|
+
summary: "Reclassify historical Cloud Run logs for a traffic source",
|
|
10044
|
+
description: 'Async one-shot backfill: pulls the last `days` of request logs (clamped server-side to the upstream retention ceiling \u2014 30d for Cloud Logging `_Default`), classifies them with the current rules, and replaces the hourly rollup buckets + sample slice in the window inside one transaction. Returns immediately with `{ runId, status: "running" }`; poll `GET /runs/{id}` for completion. lastSyncedAt only advances forward, so a backfill never undoes incremental sync progress that ran ahead of it.',
|
|
10045
|
+
tags: ["traffic"],
|
|
10046
|
+
parameters: [
|
|
10047
|
+
nameParameter,
|
|
10048
|
+
{ name: "id", in: "path", required: true, description: "Traffic source ID.", schema: stringSchema }
|
|
10049
|
+
],
|
|
10050
|
+
requestBody: {
|
|
10051
|
+
required: false,
|
|
10052
|
+
content: {
|
|
10053
|
+
"application/json": {
|
|
10054
|
+
schema: {
|
|
10055
|
+
type: "object",
|
|
10056
|
+
properties: {
|
|
10057
|
+
days: { ...integerSchema, description: "Lookback window in days (default 30, capped at the upstream retention ceiling)." }
|
|
10058
|
+
}
|
|
10059
|
+
}
|
|
10060
|
+
}
|
|
10061
|
+
}
|
|
10062
|
+
},
|
|
10063
|
+
responses: {
|
|
10064
|
+
200: { description: "Backfill submitted; poll the returned runId for completion." },
|
|
10065
|
+
400: { description: "Invalid backfill request or missing credentials." },
|
|
10066
|
+
404: { description: "Project or traffic source not found." }
|
|
10067
|
+
}
|
|
10068
|
+
},
|
|
10040
10069
|
{
|
|
10041
10070
|
method: "get",
|
|
10042
10071
|
path: "/api/v1/projects/{name}/traffic/sources",
|
|
@@ -13008,7 +13037,7 @@ async function bingRoutes(app, opts) {
|
|
|
13008
13037
|
impressions: s.Impressions,
|
|
13009
13038
|
clicks: s.Clicks,
|
|
13010
13039
|
ctr: s.Impressions > 0 ? s.Clicks / s.Impressions : 0,
|
|
13011
|
-
averagePosition: s.
|
|
13040
|
+
averagePosition: s.AvgClickPosition > 0 ? s.AvgClickPosition : s.AvgImpressionPosition > 0 ? s.AvgImpressionPosition : 0
|
|
13012
13041
|
}));
|
|
13013
13042
|
});
|
|
13014
13043
|
}
|
|
@@ -16584,11 +16613,12 @@ async function listCloudRunTrafficEvents(accessToken, options) {
|
|
|
16584
16613
|
let rawEntryCount = 0;
|
|
16585
16614
|
let skippedEntryCount = 0;
|
|
16586
16615
|
const events = [];
|
|
16616
|
+
const orderBy = options.orderBy ?? (options.firstSync ? "timestamp desc" : "timestamp asc");
|
|
16587
16617
|
for (let page = 0; page < maxPages; page += 1) {
|
|
16588
16618
|
const requestBody = {
|
|
16589
16619
|
resourceNames: [`projects/${options.gcpProjectId}`],
|
|
16590
16620
|
filter,
|
|
16591
|
-
orderBy
|
|
16621
|
+
orderBy,
|
|
16592
16622
|
pageSize
|
|
16593
16623
|
};
|
|
16594
16624
|
if (pageToken) {
|
|
@@ -16763,6 +16793,14 @@ function utmSourceFromQuery(queryString) {
|
|
|
16763
16793
|
const source = params.get("utm_source");
|
|
16764
16794
|
return source ? normalizeHost(source) : null;
|
|
16765
16795
|
}
|
|
16796
|
+
function utmSourceFromUrl(value) {
|
|
16797
|
+
if (!value) return null;
|
|
16798
|
+
try {
|
|
16799
|
+
return utmSourceFromQuery(new URL(value).search.replace(/^\?/, ""));
|
|
16800
|
+
} catch {
|
|
16801
|
+
return null;
|
|
16802
|
+
}
|
|
16803
|
+
}
|
|
16766
16804
|
function classifyCrawler(event) {
|
|
16767
16805
|
const userAgent = event.userAgent?.trim();
|
|
16768
16806
|
if (!userAgent) return null;
|
|
@@ -16805,6 +16843,18 @@ function classifyAiReferral(event) {
|
|
|
16805
16843
|
};
|
|
16806
16844
|
}
|
|
16807
16845
|
}
|
|
16846
|
+
const refererUtmSource = utmSourceFromUrl(event.referer);
|
|
16847
|
+
if (refererUtmSource) {
|
|
16848
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(refererUtmSource, candidate.domain));
|
|
16849
|
+
if (rule) {
|
|
16850
|
+
return {
|
|
16851
|
+
operator: rule.operator,
|
|
16852
|
+
product: rule.product,
|
|
16853
|
+
sourceDomain: refererUtmSource,
|
|
16854
|
+
evidenceType: "referer-utm"
|
|
16855
|
+
};
|
|
16856
|
+
}
|
|
16857
|
+
}
|
|
16808
16858
|
return null;
|
|
16809
16859
|
}
|
|
16810
16860
|
|
|
@@ -16965,6 +17015,10 @@ var DEFAULT_PAGE_SIZE2 = 1e3;
|
|
|
16965
17015
|
var DEFAULT_MAX_PAGES2 = 5;
|
|
16966
17016
|
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
16967
17017
|
var MAX_TRACKED_EVENT_IDS = 1e3;
|
|
17018
|
+
var DEFAULT_BACKFILL_DAYS = 30;
|
|
17019
|
+
var MAX_BACKFILL_DAYS = 30;
|
|
17020
|
+
var BACKFILL_MAX_PAGES = 1e3;
|
|
17021
|
+
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
16968
17022
|
function parseSourceConfig(row) {
|
|
16969
17023
|
return parseJsonColumn(row.configJson, {});
|
|
16970
17024
|
}
|
|
@@ -16995,6 +17049,173 @@ async function defaultResolveAccessToken(record) {
|
|
|
16995
17049
|
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
16996
17050
|
);
|
|
16997
17051
|
}
|
|
17052
|
+
async function runBackfillTask(options) {
|
|
17053
|
+
const {
|
|
17054
|
+
app,
|
|
17055
|
+
runId,
|
|
17056
|
+
project,
|
|
17057
|
+
sourceRow,
|
|
17058
|
+
gcpProjectId,
|
|
17059
|
+
serviceName,
|
|
17060
|
+
location,
|
|
17061
|
+
credential,
|
|
17062
|
+
windowStart,
|
|
17063
|
+
windowEnd,
|
|
17064
|
+
pullEvents,
|
|
17065
|
+
resolveAccessToken: resolveAccessToken2
|
|
17066
|
+
} = options;
|
|
17067
|
+
const markFailed = (msg) => {
|
|
17068
|
+
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17069
|
+
try {
|
|
17070
|
+
app.db.transaction((tx) => {
|
|
17071
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq23(runs.id, runId)).run();
|
|
17072
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
17073
|
+
});
|
|
17074
|
+
} catch {
|
|
17075
|
+
}
|
|
17076
|
+
};
|
|
17077
|
+
let accessToken;
|
|
17078
|
+
try {
|
|
17079
|
+
accessToken = await resolveAccessToken2(credential);
|
|
17080
|
+
} catch (e) {
|
|
17081
|
+
markFailed(`Failed to resolve Cloud Run access token: ${e instanceof Error ? e.message : String(e)}`);
|
|
17082
|
+
return;
|
|
17083
|
+
}
|
|
17084
|
+
const allEvents = [];
|
|
17085
|
+
try {
|
|
17086
|
+
const page = await pullEvents(accessToken, {
|
|
17087
|
+
gcpProjectId,
|
|
17088
|
+
serviceName,
|
|
17089
|
+
location,
|
|
17090
|
+
startTime: windowStart.toISOString(),
|
|
17091
|
+
endTime: windowEnd.toISOString(),
|
|
17092
|
+
pageSize: DEFAULT_PAGE_SIZE2,
|
|
17093
|
+
maxPages: BACKFILL_MAX_PAGES,
|
|
17094
|
+
// Backfill is intentionally `firstSync: false`. We don't want desc
|
|
17095
|
+
// ordering — the in-memory rollup builder handles any order, and the
|
|
17096
|
+
// ring-buffer reseed at the end takes the most-recent IDs from the
|
|
17097
|
+
// dedupedEvents anyway.
|
|
17098
|
+
firstSync: false,
|
|
17099
|
+
orderBy: "timestamp asc"
|
|
17100
|
+
});
|
|
17101
|
+
allEvents.push(...page.events);
|
|
17102
|
+
} catch (e) {
|
|
17103
|
+
markFailed(`Cloud Run pull failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
17104
|
+
return;
|
|
17105
|
+
}
|
|
17106
|
+
if (allEvents.length === 0) {
|
|
17107
|
+
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
17108
|
+
try {
|
|
17109
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(eq23(runs.id, runId)).run();
|
|
17110
|
+
} catch {
|
|
17111
|
+
}
|
|
17112
|
+
return;
|
|
17113
|
+
}
|
|
17114
|
+
const report = buildTrafficProbeReport(allEvents, { sampleLimit: BACKFILL_SAMPLE_LIMIT });
|
|
17115
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
17116
|
+
const windowStartIso = windowStart.toISOString();
|
|
17117
|
+
const windowEndIso = windowEnd.toISOString();
|
|
17118
|
+
const newSorted = allEvents.slice().sort((a, b) => a.observedAt < b.observedAt ? 1 : a.observedAt > b.observedAt ? -1 : 0).map((e) => e.eventId);
|
|
17119
|
+
const newRingBuffer = newSorted.slice(0, MAX_TRACKED_EVENT_IDS);
|
|
17120
|
+
const currentLastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
17121
|
+
const nextLastSyncedAt = Math.max(currentLastSyncedMs, windowEnd.getTime()) === windowEnd.getTime() ? finishedAt : sourceRow.lastSyncedAt;
|
|
17122
|
+
try {
|
|
17123
|
+
app.db.transaction((tx) => {
|
|
17124
|
+
tx.delete(crawlerEventsHourly).where(
|
|
17125
|
+
and14(
|
|
17126
|
+
eq23(crawlerEventsHourly.sourceId, sourceRow.id),
|
|
17127
|
+
gte2(crawlerEventsHourly.tsHour, windowStartIso),
|
|
17128
|
+
lte2(crawlerEventsHourly.tsHour, windowEndIso)
|
|
17129
|
+
)
|
|
17130
|
+
).run();
|
|
17131
|
+
tx.delete(aiReferralEventsHourly).where(
|
|
17132
|
+
and14(
|
|
17133
|
+
eq23(aiReferralEventsHourly.sourceId, sourceRow.id),
|
|
17134
|
+
gte2(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
17135
|
+
lte2(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
17136
|
+
)
|
|
17137
|
+
).run();
|
|
17138
|
+
tx.delete(rawEventSamples).where(
|
|
17139
|
+
and14(
|
|
17140
|
+
eq23(rawEventSamples.sourceId, sourceRow.id),
|
|
17141
|
+
gte2(rawEventSamples.ts, windowStartIso),
|
|
17142
|
+
lte2(rawEventSamples.ts, windowEndIso)
|
|
17143
|
+
)
|
|
17144
|
+
).run();
|
|
17145
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
17146
|
+
tx.insert(crawlerEventsHourly).values({
|
|
17147
|
+
projectId: project.id,
|
|
17148
|
+
sourceId: sourceRow.id,
|
|
17149
|
+
tsHour: bucket.tsHour,
|
|
17150
|
+
botId: bucket.botId,
|
|
17151
|
+
operator: bucket.operator,
|
|
17152
|
+
verificationStatus: bucket.verificationStatus,
|
|
17153
|
+
pathNormalized: bucket.pathNormalized,
|
|
17154
|
+
status: bucket.status ?? 0,
|
|
17155
|
+
hits: bucket.hits,
|
|
17156
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
17157
|
+
createdAt: finishedAt,
|
|
17158
|
+
updatedAt: finishedAt
|
|
17159
|
+
}).run();
|
|
17160
|
+
}
|
|
17161
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
17162
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
17163
|
+
projectId: project.id,
|
|
17164
|
+
sourceId: sourceRow.id,
|
|
17165
|
+
tsHour: bucket.tsHour,
|
|
17166
|
+
product: bucket.product,
|
|
17167
|
+
operator: bucket.operator,
|
|
17168
|
+
sourceDomain: bucket.sourceDomain,
|
|
17169
|
+
evidenceType: bucket.evidenceType,
|
|
17170
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
17171
|
+
status: bucket.status ?? 0,
|
|
17172
|
+
sessionsOrHits: bucket.hits,
|
|
17173
|
+
usersEstimated: null,
|
|
17174
|
+
createdAt: finishedAt,
|
|
17175
|
+
updatedAt: finishedAt
|
|
17176
|
+
}).run();
|
|
17177
|
+
}
|
|
17178
|
+
for (const sample of report.samples) {
|
|
17179
|
+
const eventType = sample.crawler ? "crawler" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
17180
|
+
const refererHost = (() => {
|
|
17181
|
+
if (!sample.referer) return null;
|
|
17182
|
+
try {
|
|
17183
|
+
return new URL(sample.referer).hostname;
|
|
17184
|
+
} catch {
|
|
17185
|
+
return null;
|
|
17186
|
+
}
|
|
17187
|
+
})();
|
|
17188
|
+
tx.insert(rawEventSamples).values({
|
|
17189
|
+
id: crypto20.randomUUID(),
|
|
17190
|
+
projectId: project.id,
|
|
17191
|
+
sourceId: sourceRow.id,
|
|
17192
|
+
ts: sample.observedAt,
|
|
17193
|
+
eventType,
|
|
17194
|
+
ipHash: null,
|
|
17195
|
+
userAgent: sample.userAgent,
|
|
17196
|
+
pathNormalized: sample.pathNormalized,
|
|
17197
|
+
status: sample.status,
|
|
17198
|
+
refererHost,
|
|
17199
|
+
classifierDetailsJson: JSON.stringify({
|
|
17200
|
+
crawler: sample.crawler,
|
|
17201
|
+
aiReferral: sample.aiReferral
|
|
17202
|
+
}),
|
|
17203
|
+
createdAt: finishedAt
|
|
17204
|
+
}).run();
|
|
17205
|
+
}
|
|
17206
|
+
tx.update(trafficSources).set({
|
|
17207
|
+
status: TrafficSourceStatuses.connected,
|
|
17208
|
+
lastSyncedAt: nextLastSyncedAt,
|
|
17209
|
+
lastError: null,
|
|
17210
|
+
lastEventIds: JSON.stringify(newRingBuffer),
|
|
17211
|
+
updatedAt: finishedAt
|
|
17212
|
+
}).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
17213
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
17214
|
+
});
|
|
17215
|
+
} catch (e) {
|
|
17216
|
+
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
17217
|
+
}
|
|
17218
|
+
}
|
|
16998
17219
|
async function trafficRoutes(app, opts) {
|
|
16999
17220
|
const pullEvents = opts.pullCloudRunEvents ?? listCloudRunTrafficEvents;
|
|
17000
17221
|
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
@@ -17158,6 +17379,7 @@ async function trafficRoutes(app, opts) {
|
|
|
17158
17379
|
markFailed(msg, "PROVIDER_AUTH");
|
|
17159
17380
|
throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
17160
17381
|
}
|
|
17382
|
+
const isFirstSync = !sourceRow.lastSyncedAt;
|
|
17161
17383
|
let allEvents = [];
|
|
17162
17384
|
try {
|
|
17163
17385
|
const page = await pullEvents(accessToken, {
|
|
@@ -17167,7 +17389,8 @@ async function trafficRoutes(app, opts) {
|
|
|
17167
17389
|
startTime: windowStart.toISOString(),
|
|
17168
17390
|
endTime: windowEnd.toISOString(),
|
|
17169
17391
|
pageSize,
|
|
17170
|
-
maxPages
|
|
17392
|
+
maxPages,
|
|
17393
|
+
firstSync: isFirstSync
|
|
17171
17394
|
});
|
|
17172
17395
|
allEvents = page.events;
|
|
17173
17396
|
} catch (e) {
|
|
@@ -17334,6 +17557,78 @@ async function trafficRoutes(app, opts) {
|
|
|
17334
17557
|
};
|
|
17335
17558
|
return response;
|
|
17336
17559
|
});
|
|
17560
|
+
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
17561
|
+
const project = resolveProject(app.db, request.params.name);
|
|
17562
|
+
const sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, request.params.id)).get();
|
|
17563
|
+
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
17564
|
+
throw notFound("Traffic source", request.params.id);
|
|
17565
|
+
}
|
|
17566
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
17567
|
+
throw validationError(
|
|
17568
|
+
`Backfill for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run is supported in v1.`
|
|
17569
|
+
);
|
|
17570
|
+
}
|
|
17571
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
17572
|
+
if (!credentialStore) {
|
|
17573
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
17574
|
+
}
|
|
17575
|
+
const credential = credentialStore.getConnection(project.name);
|
|
17576
|
+
if (!credential) {
|
|
17577
|
+
throw validationError(
|
|
17578
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
17579
|
+
);
|
|
17580
|
+
}
|
|
17581
|
+
const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
|
|
17582
|
+
if (!Number.isInteger(requestedDays) || requestedDays <= 0) {
|
|
17583
|
+
throw validationError('"days" must be a positive integer');
|
|
17584
|
+
}
|
|
17585
|
+
const appliedDays = Math.min(requestedDays, MAX_BACKFILL_DAYS);
|
|
17586
|
+
const config = parseSourceConfig(sourceRow);
|
|
17587
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
17588
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
17589
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
17590
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
17591
|
+
const windowStart = new Date(windowEnd.getTime() - appliedDays * 864e5);
|
|
17592
|
+
windowStart.setUTCMinutes(0, 0, 0);
|
|
17593
|
+
const startedAt = windowEnd.toISOString();
|
|
17594
|
+
const runId = crypto20.randomUUID();
|
|
17595
|
+
app.db.insert(runs).values({
|
|
17596
|
+
id: runId,
|
|
17597
|
+
projectId: project.id,
|
|
17598
|
+
kind: RunKinds["traffic-sync"],
|
|
17599
|
+
status: RunStatuses.running,
|
|
17600
|
+
trigger: RunTriggers.backfill,
|
|
17601
|
+
sourceId: sourceRow.id,
|
|
17602
|
+
startedAt,
|
|
17603
|
+
createdAt: startedAt
|
|
17604
|
+
}).run();
|
|
17605
|
+
void runBackfillTask({
|
|
17606
|
+
app,
|
|
17607
|
+
runId,
|
|
17608
|
+
project,
|
|
17609
|
+
sourceRow,
|
|
17610
|
+
gcpProjectId,
|
|
17611
|
+
serviceName,
|
|
17612
|
+
location,
|
|
17613
|
+
credential,
|
|
17614
|
+
windowStart,
|
|
17615
|
+
windowEnd,
|
|
17616
|
+
appliedDays,
|
|
17617
|
+
pullEvents,
|
|
17618
|
+
resolveAccessToken: resolveAccessToken2
|
|
17619
|
+
}).catch(() => {
|
|
17620
|
+
});
|
|
17621
|
+
const response = {
|
|
17622
|
+
sourceId: sourceRow.id,
|
|
17623
|
+
runId,
|
|
17624
|
+
status: RunStatuses.running,
|
|
17625
|
+
windowStart: windowStart.toISOString(),
|
|
17626
|
+
windowEnd: windowEnd.toISOString(),
|
|
17627
|
+
daysRequested: requestedDays,
|
|
17628
|
+
daysApplied: appliedDays
|
|
17629
|
+
};
|
|
17630
|
+
return response;
|
|
17631
|
+
});
|
|
17337
17632
|
function buildSourceDetail(projectId, row, since) {
|
|
17338
17633
|
const crawlerTotals = app.db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
17339
17634
|
and14(
|
|
@@ -91,7 +91,7 @@ var runKindSchema = z2.enum([
|
|
|
91
91
|
"traffic-sync"
|
|
92
92
|
]);
|
|
93
93
|
var RunKinds = runKindSchema.enum;
|
|
94
|
-
var runTriggerSchema = z2.enum(["manual", "scheduled", "config-apply"]);
|
|
94
|
+
var runTriggerSchema = z2.enum(["manual", "scheduled", "config-apply", "backfill"]);
|
|
95
95
|
var RunTriggers = runTriggerSchema.enum;
|
|
96
96
|
var citationStateSchema = z2.enum(["cited", "not-cited"]);
|
|
97
97
|
var CitationStates = citationStateSchema.enum;
|
|
@@ -2263,6 +2263,20 @@ var trafficSyncResponseSchema = z20.object({
|
|
|
2263
2263
|
windowStart: z20.string(),
|
|
2264
2264
|
windowEnd: z20.string()
|
|
2265
2265
|
});
|
|
2266
|
+
var trafficBackfillRequestSchema = z20.object({
|
|
2267
|
+
/** Lookback window in days. Capped server-side at the upstream log retention ceiling (Cloud Logging _Default = 30d). Default: 30. */
|
|
2268
|
+
days: z20.number().int().positive().optional()
|
|
2269
|
+
});
|
|
2270
|
+
var trafficBackfillResponseSchema = z20.object({
|
|
2271
|
+
sourceId: z20.string(),
|
|
2272
|
+
runId: z20.string(),
|
|
2273
|
+
status: runStatusSchema,
|
|
2274
|
+
windowStart: z20.string(),
|
|
2275
|
+
windowEnd: z20.string(),
|
|
2276
|
+
/** Days actually used after server-side clamping (≤ requested). */
|
|
2277
|
+
daysRequested: z20.number().int().positive(),
|
|
2278
|
+
daysApplied: z20.number().int().positive()
|
|
2279
|
+
});
|
|
2266
2280
|
var trafficSourceTotalsSchema = z20.object({
|
|
2267
2281
|
crawlerHits: z20.number().int().nonnegative(),
|
|
2268
2282
|
aiReferralHits: z20.number().int().nonnegative(),
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
scheduleUpsertRequestSchema,
|
|
16
16
|
trafficConnectCloudRunRequestSchema,
|
|
17
17
|
trafficEventKindSchema
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-EY63PENL.js";
|
|
19
19
|
|
|
20
20
|
// src/config.ts
|
|
21
21
|
import fs from "fs";
|
|
@@ -773,6 +773,13 @@ var ApiClient = class {
|
|
|
773
773
|
body ?? {}
|
|
774
774
|
);
|
|
775
775
|
}
|
|
776
|
+
async trafficBackfill(project, sourceId, body) {
|
|
777
|
+
return this.request(
|
|
778
|
+
"POST",
|
|
779
|
+
`/projects/${encodeURIComponent(project)}/traffic/sources/${encodeURIComponent(sourceId)}/backfill`,
|
|
780
|
+
body ?? {}
|
|
781
|
+
);
|
|
782
|
+
}
|
|
776
783
|
async trafficListSources(project) {
|
|
777
784
|
return this.request(
|
|
778
785
|
"GET",
|
|
@@ -1190,6 +1197,11 @@ var trafficSyncInputSchema = z2.object({
|
|
|
1190
1197
|
sourceId: z2.string().min(1).describe("Traffic source ID returned by canonry_traffic_connect_cloud_run or canonry_traffic_sources_list."),
|
|
1191
1198
|
sinceMinutes: z2.number().int().positive().max(7 * 24 * 60).optional().describe("Lookback window in minutes. Defaults to the source's configured window (60 min) when omitted; clamped forward to lastSyncedAt to avoid double-counting.")
|
|
1192
1199
|
});
|
|
1200
|
+
var trafficBackfillInputSchema = z2.object({
|
|
1201
|
+
project: projectNameSchema,
|
|
1202
|
+
sourceId: z2.string().min(1).describe("Traffic source ID returned by canonry_traffic_sources_list."),
|
|
1203
|
+
days: z2.number().int().positive().max(30).optional().describe("Lookback window in days. Default 30, capped server-side at the upstream log retention ceiling (Cloud Logging _Default = 30d).")
|
|
1204
|
+
});
|
|
1193
1205
|
var trafficEventsInputSchema = z2.object({
|
|
1194
1206
|
project: projectNameSchema,
|
|
1195
1207
|
since: z2.string().optional().describe("ISO 8601 lower bound. Defaults to 24h ago when omitted."),
|
|
@@ -1788,6 +1800,17 @@ var canonryMcpTools = [
|
|
|
1788
1800
|
openApiOperations: ["POST /api/v1/projects/{name}/traffic/sources/{id}/sync"],
|
|
1789
1801
|
handler: (client, input) => client.trafficSync(input.project, input.sourceId, input.sinceMinutes !== void 0 ? { sinceMinutes: input.sinceMinutes } : void 0)
|
|
1790
1802
|
}),
|
|
1803
|
+
defineTool({
|
|
1804
|
+
name: "canonry_traffic_backfill",
|
|
1805
|
+
title: "Backfill Cloud Run traffic source",
|
|
1806
|
+
description: 'Async one-shot reclassification of historical Cloud Run logs. Pulls the last `days` of request logs (capped at the 30d Cloud Logging retention ceiling), classifies them with current rules, and replaces the hourly rollup buckets + sample slice in the window. Returns immediately with `{ runId, status: "running" }`; poll canonry_run_get for completion. lastSyncedAt only advances forward \u2014 a backfill never undoes incremental sync progress that ran ahead of it.',
|
|
1807
|
+
access: "write",
|
|
1808
|
+
tier: "traffic",
|
|
1809
|
+
inputSchema: trafficBackfillInputSchema,
|
|
1810
|
+
annotations: writeAnnotations({ idempotentHint: true, destructiveHint: true, openWorldHint: true }),
|
|
1811
|
+
openApiOperations: ["POST /api/v1/projects/{name}/traffic/sources/{id}/backfill"],
|
|
1812
|
+
handler: (client, input) => client.trafficBackfill(input.project, input.sourceId, input.days !== void 0 ? { days: input.days } : void 0)
|
|
1813
|
+
}),
|
|
1791
1814
|
defineTool({
|
|
1792
1815
|
name: "canonry_project_upsert",
|
|
1793
1816
|
title: "Create or replace project",
|
package/dist/cli.js
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
setTelemetrySource,
|
|
21
21
|
showFirstRunNotice,
|
|
22
22
|
trackEvent
|
|
23
|
-
} from "./chunk-
|
|
23
|
+
} from "./chunk-3UGJUNQX.js";
|
|
24
24
|
import {
|
|
25
25
|
CliError,
|
|
26
26
|
EXIT_SYSTEM_ERROR,
|
|
@@ -36,7 +36,7 @@ import {
|
|
|
36
36
|
saveConfig,
|
|
37
37
|
saveConfigPatch,
|
|
38
38
|
usageError
|
|
39
|
-
} from "./chunk-
|
|
39
|
+
} from "./chunk-VFKGHXVJ.js";
|
|
40
40
|
import {
|
|
41
41
|
apiKeys,
|
|
42
42
|
competitors,
|
|
@@ -49,7 +49,7 @@ import {
|
|
|
49
49
|
queries,
|
|
50
50
|
querySnapshots,
|
|
51
51
|
runs
|
|
52
|
-
} from "./chunk-
|
|
52
|
+
} from "./chunk-GVQYROIK.js";
|
|
53
53
|
import {
|
|
54
54
|
CcReleaseSyncStatuses,
|
|
55
55
|
CheckScopes,
|
|
@@ -69,7 +69,7 @@ import {
|
|
|
69
69
|
providerQuotaPolicySchema,
|
|
70
70
|
resolveProviderInput,
|
|
71
71
|
skillsClientSchema
|
|
72
|
-
} from "./chunk-
|
|
72
|
+
} from "./chunk-EY63PENL.js";
|
|
73
73
|
|
|
74
74
|
// src/cli.ts
|
|
75
75
|
import { pathToFileURL } from "url";
|
|
@@ -621,7 +621,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
621
621
|
return result;
|
|
622
622
|
}
|
|
623
623
|
async function backfillInsightsCommand(project, opts) {
|
|
624
|
-
const { IntelligenceService } = await import("./intelligence-service-
|
|
624
|
+
const { IntelligenceService } = await import("./intelligence-service-5COCQKXG.js");
|
|
625
625
|
const config = loadConfig();
|
|
626
626
|
const db = createClient(config.database);
|
|
627
627
|
migrate(db);
|
|
@@ -2775,6 +2775,75 @@ async function trafficConnectCloudRun(project, opts) {
|
|
|
2775
2775
|
console.log("");
|
|
2776
2776
|
console.log(`Next: canonry traffic sync ${project} --source ${result.id}`);
|
|
2777
2777
|
}
|
|
2778
|
+
async function trafficBackfill(project, opts) {
|
|
2779
|
+
if (!opts.source) {
|
|
2780
|
+
throw new CliError({
|
|
2781
|
+
code: "TRAFFIC_SOURCE_REQUIRED",
|
|
2782
|
+
message: "--source <id> is required",
|
|
2783
|
+
displayMessage: "Error: --source <id> is required (run `canonry traffic sources` to list connected sources)",
|
|
2784
|
+
details: { project }
|
|
2785
|
+
});
|
|
2786
|
+
}
|
|
2787
|
+
const client = getClient5();
|
|
2788
|
+
const submitted = await client.trafficBackfill(project, opts.source, {
|
|
2789
|
+
days: opts.days
|
|
2790
|
+
});
|
|
2791
|
+
if (!opts.wait) {
|
|
2792
|
+
if (opts.format === "json") {
|
|
2793
|
+
console.log(JSON.stringify(submitted, null, 2));
|
|
2794
|
+
return;
|
|
2795
|
+
}
|
|
2796
|
+
console.log(`Backfill submitted for "${project}" (source ${opts.source}).`);
|
|
2797
|
+
console.log(` Run ID: ${submitted.runId}`);
|
|
2798
|
+
console.log(` Window: ${submitted.windowStart} \u2192 ${submitted.windowEnd}`);
|
|
2799
|
+
console.log(` Days applied: ${submitted.daysApplied} (requested ${submitted.daysRequested})`);
|
|
2800
|
+
console.log(` Status: ${submitted.status}`);
|
|
2801
|
+
console.log("");
|
|
2802
|
+
console.log(`Poll: canonry runs get ${submitted.runId}`);
|
|
2803
|
+
return;
|
|
2804
|
+
}
|
|
2805
|
+
const intervalMs = opts.pollIntervalMs ?? 1500;
|
|
2806
|
+
const deadlineMs = Date.now() + 5 * 6e4;
|
|
2807
|
+
let final = null;
|
|
2808
|
+
while (Date.now() < deadlineMs) {
|
|
2809
|
+
const run = await client.getRun(submitted.runId);
|
|
2810
|
+
if (run.status !== RunStatuses.running && run.status !== RunStatuses.queued) {
|
|
2811
|
+
final = run;
|
|
2812
|
+
break;
|
|
2813
|
+
}
|
|
2814
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
2815
|
+
}
|
|
2816
|
+
if (!final) {
|
|
2817
|
+
throw new CliError({
|
|
2818
|
+
code: "TRAFFIC_BACKFILL_TIMEOUT",
|
|
2819
|
+
message: `Backfill did not complete within 5 minutes (run ${submitted.runId} still running)`,
|
|
2820
|
+
displayMessage: `Error: backfill run ${submitted.runId} did not finish within 5 minutes \u2014 check status with "canonry runs get ${submitted.runId}"`,
|
|
2821
|
+
details: { project, runId: submitted.runId }
|
|
2822
|
+
});
|
|
2823
|
+
}
|
|
2824
|
+
if (opts.format === "json") {
|
|
2825
|
+
console.log(JSON.stringify({ ...submitted, finalStatus: final.status, finalRun: final }, null, 2));
|
|
2826
|
+
return;
|
|
2827
|
+
}
|
|
2828
|
+
if (final.status === RunStatuses.completed) {
|
|
2829
|
+
console.log(`Backfill complete for "${project}" (source ${opts.source}).`);
|
|
2830
|
+
console.log(` Run ID: ${final.id}`);
|
|
2831
|
+
console.log(` Window: ${submitted.windowStart} \u2192 ${submitted.windowEnd}`);
|
|
2832
|
+
console.log(` Days applied: ${submitted.daysApplied}`);
|
|
2833
|
+
console.log(` Started: ${final.startedAt ?? "unknown"}`);
|
|
2834
|
+
console.log(` Finished: ${final.finishedAt ?? "unknown"}`);
|
|
2835
|
+
console.log("");
|
|
2836
|
+
console.log(`Inspect rebuilt rollups: canonry traffic events ${project} --source ${opts.source} --since-minutes ${submitted.daysApplied * 24 * 60}`);
|
|
2837
|
+
return;
|
|
2838
|
+
}
|
|
2839
|
+
const errorMessage = final.error?.message ?? null;
|
|
2840
|
+
throw new CliError({
|
|
2841
|
+
code: "TRAFFIC_BACKFILL_FAILED",
|
|
2842
|
+
message: errorMessage ?? "backfill run did not complete successfully",
|
|
2843
|
+
displayMessage: `Error: backfill run ${final.id} ${final.status}${errorMessage ? ` \u2014 ${errorMessage}` : ""}`,
|
|
2844
|
+
details: { project, runId: final.id, status: final.status }
|
|
2845
|
+
});
|
|
2846
|
+
}
|
|
2778
2847
|
async function trafficSync(project, opts) {
|
|
2779
2848
|
if (!opts.source) {
|
|
2780
2849
|
throw new CliError({
|
|
@@ -2995,6 +3064,35 @@ var TRAFFIC_CLI_COMMANDS = [
|
|
|
2995
3064
|
});
|
|
2996
3065
|
}
|
|
2997
3066
|
},
|
|
3067
|
+
{
|
|
3068
|
+
path: ["traffic", "backfill"],
|
|
3069
|
+
usage: "canonry traffic backfill <project> --source <id> [--days 30] [--wait] [--format json]",
|
|
3070
|
+
options: {
|
|
3071
|
+
source: stringOption(),
|
|
3072
|
+
days: stringOption(),
|
|
3073
|
+
wait: { type: "boolean" }
|
|
3074
|
+
},
|
|
3075
|
+
run: async (input) => {
|
|
3076
|
+
const project = requireProject(
|
|
3077
|
+
input,
|
|
3078
|
+
"traffic.backfill",
|
|
3079
|
+
"canonry traffic backfill <project> --source <id> [--days 30] [--wait]"
|
|
3080
|
+
);
|
|
3081
|
+
const source = getString(input.values, "source");
|
|
3082
|
+
if (!source) throw new Error("--source <id> is required");
|
|
3083
|
+
const days = parseIntegerOption(input, "days", {
|
|
3084
|
+
command: "traffic.backfill",
|
|
3085
|
+
usage: "canonry traffic backfill <project> --source <id> [--days 30] [--wait]",
|
|
3086
|
+
message: "--days must be a positive integer"
|
|
3087
|
+
});
|
|
3088
|
+
await trafficBackfill(project, {
|
|
3089
|
+
source,
|
|
3090
|
+
days,
|
|
3091
|
+
wait: getBoolean(input.values, "wait"),
|
|
3092
|
+
format: input.format
|
|
3093
|
+
});
|
|
3094
|
+
}
|
|
3095
|
+
},
|
|
2998
3096
|
{
|
|
2999
3097
|
path: ["traffic", "sources"],
|
|
3000
3098
|
usage: "canonry traffic sources <project> [--format json]",
|
|
@@ -3064,7 +3162,7 @@ var TRAFFIC_CLI_COMMANDS = [
|
|
|
3064
3162
|
unknownSubcommand(input.positionals[0], {
|
|
3065
3163
|
command: "traffic",
|
|
3066
3164
|
usage: "canonry traffic <subcommand> <project> [args]",
|
|
3067
|
-
available: ["connect", "sync", "status", "sources", "events"]
|
|
3165
|
+
available: ["connect", "sync", "backfill", "status", "sources", "events"]
|
|
3068
3166
|
});
|
|
3069
3167
|
}
|
|
3070
3168
|
}
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createServer
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-3UGJUNQX.js";
|
|
4
4
|
import {
|
|
5
5
|
loadConfig
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-VFKGHXVJ.js";
|
|
7
|
+
import "./chunk-GVQYROIK.js";
|
|
8
|
+
import "./chunk-EY63PENL.js";
|
|
9
9
|
export {
|
|
10
10
|
createServer,
|
|
11
11
|
loadConfig
|