@canonry/canonry 6.1.4 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +1 -1
- package/assets/assets/{AuditHistoryPanel-D-W_kwEY.js → AuditHistoryPanel-CuEMfkaA.js} +1 -1
- package/assets/assets/{BacklinksPage-BeqCFRO3.js → BacklinksPage-CftadZ1-.js} +1 -1
- package/assets/assets/{HistoryPage-wAPbY0z4.js → HistoryPage-DouRgshN.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BSOrkHCB.js → MeasurementPropertyPage-qfqv2qs9.js} +1 -1
- package/assets/assets/{ProjectPage-v0C7CwyA.js → ProjectPage-DqEVwe4t.js} +1 -1
- package/assets/assets/{RunsPage-CxolaIps.js → RunsPage-CUkMs0gB.js} +1 -1
- package/assets/assets/{SettingsPage-BQqJz0Ms.js → SettingsPage-LLQ40tSh.js} +1 -1
- package/assets/assets/{SiteHealthSection-C56kNcQQ.js → SiteHealthSection-Ba7foDiU.js} +3 -3
- package/assets/assets/{TrafficPage-BDmWdYWf.js → TrafficPage-Cu_ZR3--.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DZl3YNBs.js → TrafficSourceDetailPage-NKZq7jf9.js} +1 -1
- package/assets/assets/{extract-error-message-D0mBVEGn.js → extract-error-message-BdQYrz9N.js} +1 -1
- package/assets/assets/{index-DZ__RzGa.js → index-HOeLz1ZL.js} +20 -20
- package/assets/assets/{react-sigma_core.esm.min-BQNp33-g.js → react-sigma_core.esm.min-CZaoE5cU.js} +1 -1
- package/assets/assets/{v2-overview-adapter-XZbD7swG.js → v2-overview-adapter-BVx4nI49.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-5LZLDXEN.js → chunk-2RFHQ6XT.js} +396 -172
- package/dist/{chunk-HB6XZOZ6.js → chunk-4EKOKWQT.js} +2 -2
- package/dist/{chunk-4MWLHTRS.js → chunk-6EPN4XY5.js} +13 -0
- package/dist/{chunk-N7MRAN3G.js → chunk-ENI3GOWW.js} +24 -5
- package/dist/{chunk-YOMLHHJS.js → chunk-V6JUBTNL.js} +3 -3
- package/dist/{chunk-6CR7DWLU.js → chunk-W3GDZZNF.js} +1 -1
- package/dist/{chunk-TBAVVIVI.js → chunk-Y2B7JRSB.js} +3 -3
- package/dist/cli.js +45 -16
- package/dist/{demo-server-2622GU2H.js → demo-server-CH7MOBXW.js} +4 -4
- package/dist/index.js +6 -6
- package/dist/{intelligence-service-QUAOWEIA.js → intelligence-service-4IONMOFB.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +10 -10
|
@@ -19,21 +19,21 @@ import {
|
|
|
19
19
|
runWithUsageTags,
|
|
20
20
|
saveConfigPatch,
|
|
21
21
|
sentimentInstallReadiness
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-V6JUBTNL.js";
|
|
23
23
|
import {
|
|
24
24
|
computeFactorAverages,
|
|
25
25
|
deleteSiteCrawlGraphLayout,
|
|
26
26
|
persistSiteCrawlGraphLayout,
|
|
27
27
|
prepareSiteCrawlGraphLayout,
|
|
28
28
|
recordedSharePct
|
|
29
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-4EKOKWQT.js";
|
|
30
30
|
import {
|
|
31
31
|
CanonryMcpToolNames,
|
|
32
32
|
CliError,
|
|
33
33
|
PACKAGE_VERSION,
|
|
34
34
|
canonryMcpTools,
|
|
35
35
|
isMachineFormat
|
|
36
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-W3GDZZNF.js";
|
|
37
37
|
import {
|
|
38
38
|
CC_CACHE_DIR,
|
|
39
39
|
DUCKDB_SPEC,
|
|
@@ -214,7 +214,7 @@ import {
|
|
|
214
214
|
siteLivenessState,
|
|
215
215
|
toAlertView,
|
|
216
216
|
usageCounters
|
|
217
|
-
} from "./chunk-
|
|
217
|
+
} from "./chunk-ENI3GOWW.js";
|
|
218
218
|
import {
|
|
219
219
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
220
220
|
AGENT_NONE,
|
|
@@ -304,6 +304,7 @@ import {
|
|
|
304
304
|
canonicalizeGtmAccountId,
|
|
305
305
|
canonicalizeGtmContainerId,
|
|
306
306
|
canonicalizeGtmResourceSelection,
|
|
307
|
+
classifyProviderErrorMessage,
|
|
307
308
|
classifyProviderErrorMessages,
|
|
308
309
|
classifyTemplateLinkEdge,
|
|
309
310
|
cloudflareQueueNameSchema,
|
|
@@ -327,6 +328,7 @@ import {
|
|
|
327
328
|
effectiveBrandNames,
|
|
328
329
|
effectiveDomains,
|
|
329
330
|
embedClientConfigForRequest,
|
|
331
|
+
extractProviderHttpStatus,
|
|
330
332
|
factorStatusFromScore,
|
|
331
333
|
filterCapturedCitedUrls,
|
|
332
334
|
forbidden,
|
|
@@ -414,7 +416,7 @@ import {
|
|
|
414
416
|
visibilityReportRequestSchema,
|
|
415
417
|
winnabilityClassLabel,
|
|
416
418
|
withRetry
|
|
417
|
-
} from "./chunk-
|
|
419
|
+
} from "./chunk-6EPN4XY5.js";
|
|
418
420
|
|
|
419
421
|
// src/runtime-context.ts
|
|
420
422
|
function cliRuntimeContext(env = process.env, stdio = process) {
|
|
@@ -4053,6 +4055,19 @@ import os4 from "os";
|
|
|
4053
4055
|
import { GoogleGenAI } from "@google/genai";
|
|
4054
4056
|
|
|
4055
4057
|
// ../provider-gemini/src/utils.ts
|
|
4058
|
+
var GEMINI_MAX_RATE_LIMIT_WAIT_MS = 65e3;
|
|
4059
|
+
function geminiRetryDelayMs(err) {
|
|
4060
|
+
const match = /"retryDelay"\s*:\s*"(\d+(?:\.\d+)?)s"/.exec(describeError(err));
|
|
4061
|
+
return match ? Math.round(Number(match[1]) * 1e3) : null;
|
|
4062
|
+
}
|
|
4063
|
+
function isGeminiDailyQuotaError(err) {
|
|
4064
|
+
return /"quotaId"\s*:\s*"[^"]*PerDay/i.test(describeError(err));
|
|
4065
|
+
}
|
|
4066
|
+
function isGeminiRetryable(err) {
|
|
4067
|
+
if (!isRetryableHttpError(err) || isGeminiDailyQuotaError(err)) return false;
|
|
4068
|
+
const asked = geminiRetryDelayMs(err);
|
|
4069
|
+
return asked === null || asked <= GEMINI_MAX_RATE_LIMIT_WAIT_MS;
|
|
4070
|
+
}
|
|
4056
4071
|
async function withRetry2(fn, options = {}) {
|
|
4057
4072
|
return withRetry(fn, {
|
|
4058
4073
|
maxRetries: options.maxRetries ?? 3,
|
|
@@ -4061,7 +4076,12 @@ async function withRetry2(fn, options = {}) {
|
|
|
4061
4076
|
// provider has shipped to date (1000, 2000, 4000ms). Enable per-caller
|
|
4062
4077
|
// if you need it.
|
|
4063
4078
|
jitter: false,
|
|
4064
|
-
isRetryable:
|
|
4079
|
+
isRetryable: isGeminiRetryable,
|
|
4080
|
+
computeDelayMs: (_attempt, err, defaultMs) => {
|
|
4081
|
+
const asked = geminiRetryDelayMs(err);
|
|
4082
|
+
return asked === null ? defaultMs : asked + 500;
|
|
4083
|
+
},
|
|
4084
|
+
...options.sleep ? { sleep: options.sleep } : {},
|
|
4065
4085
|
onRetry: ({ attempt, err, delayMs }) => {
|
|
4066
4086
|
console.warn(
|
|
4067
4087
|
`[provider] Attempt ${attempt + 1} failed, retrying in ${delayMs}ms...`,
|
|
@@ -4072,7 +4092,7 @@ async function withRetry2(fn, options = {}) {
|
|
|
4072
4092
|
}
|
|
4073
4093
|
|
|
4074
4094
|
// ../provider-gemini/src/normalize.ts
|
|
4075
|
-
var DEFAULT_MODEL = "gemini-
|
|
4095
|
+
var DEFAULT_MODEL = "gemini-flash-latest";
|
|
4076
4096
|
function isVertexConfig(config) {
|
|
4077
4097
|
return !!config.vertexProject;
|
|
4078
4098
|
}
|
|
@@ -4515,10 +4535,20 @@ var geminiAdapter = {
|
|
|
4515
4535
|
keyUrl: "https://aistudio.google.com/apikey",
|
|
4516
4536
|
// Upstream model list: https://ai.google.dev/gemini-api/docs/models
|
|
4517
4537
|
modelRegistry: {
|
|
4518
|
-
defaultModel: "gemini-
|
|
4538
|
+
defaultModel: "gemini-flash-latest",
|
|
4519
4539
|
validationPattern: /./,
|
|
4520
|
-
validationHint: "any valid Google model name (e.g. gemini-
|
|
4540
|
+
validationHint: "any valid Google model name (e.g. gemini-flash-latest, gemini-3.8-flash)",
|
|
4521
4541
|
knownModels: [
|
|
4542
|
+
// Google's moving aliases: always the newest Flash / Flash-Lite, with
|
|
4543
|
+
// 2 weeks' email notice before a breaking change. The model actually
|
|
4544
|
+
// served is recorded per snapshot (`servedModel`), so trends still show
|
|
4545
|
+
// when the model behind an alias changed.
|
|
4546
|
+
{ id: "gemini-flash-latest", displayName: "Gemini Flash (latest)", tier: "standard" },
|
|
4547
|
+
{ id: "gemini-flash-lite-latest", displayName: "Gemini Flash-Lite (latest)", tier: "economy" },
|
|
4548
|
+
{ id: "gemini-3.8-flash", displayName: "Gemini 3.8 Flash", tier: "standard" },
|
|
4549
|
+
{ id: "gemini-3.5-flash-lite", displayName: "Gemini 3.5 Flash-Lite", tier: "economy" },
|
|
4550
|
+
// Google limits the 2.5 models to projects that already used them, so a
|
|
4551
|
+
// new key may be refused. Listed for installs that still pin them.
|
|
4522
4552
|
{ id: "gemini-2.5-pro", displayName: "Gemini 2.5 Pro", tier: "flagship" },
|
|
4523
4553
|
{ id: "gemini-2.5-flash", displayName: "Gemini 2.5 Flash", tier: "standard" },
|
|
4524
4554
|
{ id: "gemini-2.5-flash-lite", displayName: "Gemini 2.5 Flash-Lite", tier: "economy" },
|
|
@@ -7830,7 +7860,7 @@ function nonBlank(value) {
|
|
|
7830
7860
|
return trimmed ? trimmed : void 0;
|
|
7831
7861
|
}
|
|
7832
7862
|
function resolveBuildCommit(env = process.env) {
|
|
7833
|
-
const embedded = true ? "
|
|
7863
|
+
const embedded = true ? "633bd63832cf6af1da610e63f376dfc872975b49" : void 0;
|
|
7834
7864
|
return nonBlank(embedded) ?? nonBlank(env.CANONRY_COMMIT);
|
|
7835
7865
|
}
|
|
7836
7866
|
function resolveInstanceIdentity(env = process.env) {
|
|
@@ -7845,7 +7875,8 @@ import crypto6 from "crypto";
|
|
|
7845
7875
|
import fs4 from "fs";
|
|
7846
7876
|
import path3 from "path";
|
|
7847
7877
|
import os3 from "os";
|
|
7848
|
-
import {
|
|
7878
|
+
import { setImmediate as yieldToEventLoop } from "timers/promises";
|
|
7879
|
+
import { and as and3, asc as asc2, count, desc, eq as eq3, inArray, isNotNull, isNull, lt, ne, sql as sql2 } from "drizzle-orm";
|
|
7849
7880
|
|
|
7850
7881
|
// src/run-telemetry.ts
|
|
7851
7882
|
import crypto4 from "crypto";
|
|
@@ -7910,6 +7941,51 @@ function buildSiteAuditCompletedProps(input) {
|
|
|
7910
7941
|
if (crawl.checkDeadLinks) props.deadLinksFound = crawl.deadLinksFound;
|
|
7911
7942
|
return props;
|
|
7912
7943
|
}
|
|
7944
|
+
var MAX_NESTED_KEYS = 12;
|
|
7945
|
+
function buildProviderOutcomeProps(providers, providerErrors) {
|
|
7946
|
+
if (providerErrors.size === 0) return {};
|
|
7947
|
+
const names = [.../* @__PURE__ */ new Set([...providers, ...providerErrors.keys()])].slice(0, MAX_NESTED_KEYS);
|
|
7948
|
+
const providerOutcomes = {};
|
|
7949
|
+
const providerHttpStatus = {};
|
|
7950
|
+
for (const name of names) {
|
|
7951
|
+
const message = providerErrors.get(name);
|
|
7952
|
+
if (message === void 0) {
|
|
7953
|
+
providerOutcomes[name] = "ok";
|
|
7954
|
+
continue;
|
|
7955
|
+
}
|
|
7956
|
+
providerOutcomes[name] = classifyProviderErrorMessage(message);
|
|
7957
|
+
const status = extractProviderHttpStatus(message);
|
|
7958
|
+
if (status !== void 0) providerHttpStatus[name] = status;
|
|
7959
|
+
}
|
|
7960
|
+
return Object.keys(providerHttpStatus).length > 0 ? { providerOutcomes, providerHttpStatus } : { providerOutcomes };
|
|
7961
|
+
}
|
|
7962
|
+
function runFailureSite(providerCallStart, providerCallEnd) {
|
|
7963
|
+
if (providerCallStart === void 0) return "setup";
|
|
7964
|
+
return providerCallEnd === void 0 ? "provider_call" : "finalize";
|
|
7965
|
+
}
|
|
7966
|
+
var ERROR_NAME_RE = /^[a-z_$][\w$]{0,39}$/i;
|
|
7967
|
+
var ERROR_SYS_CODE_RE = /^[A-Z][A-Z0-9_]{1,39}$/;
|
|
7968
|
+
function describeRunFailure(err, site) {
|
|
7969
|
+
const props = site ? { errorSite: site } : {};
|
|
7970
|
+
const name = err instanceof Error ? err.name : typeof err;
|
|
7971
|
+
if (ERROR_NAME_RE.test(name)) props.errorName = name;
|
|
7972
|
+
const code = err && typeof err === "object" && "code" in err ? err.code : void 0;
|
|
7973
|
+
if (typeof code === "string" && ERROR_SYS_CODE_RE.test(code)) props.errorSysCode = code;
|
|
7974
|
+
return props;
|
|
7975
|
+
}
|
|
7976
|
+
var FAILURE_STREAK_SAMPLE_AFTER = 5;
|
|
7977
|
+
var FAILURE_STREAK_SAMPLE_RATE = 20;
|
|
7978
|
+
function failureStreakSampling(status, priorStreak, runId) {
|
|
7979
|
+
if (status !== "failed" && status !== "partial") return { report: true, props: {} };
|
|
7980
|
+
if (priorStreak < FAILURE_STREAK_SAMPLE_AFTER) {
|
|
7981
|
+
return { report: true, props: { failureStreak: priorStreak } };
|
|
7982
|
+
}
|
|
7983
|
+
const bucket = crypto4.createHash("sha256").update(runId).digest().readUInt32BE(0) % FAILURE_STREAK_SAMPLE_RATE;
|
|
7984
|
+
return {
|
|
7985
|
+
report: bucket === 0,
|
|
7986
|
+
props: { failureStreak: priorStreak, sampleRate: FAILURE_STREAK_SAMPLE_RATE }
|
|
7987
|
+
};
|
|
7988
|
+
}
|
|
7913
7989
|
|
|
7914
7990
|
// src/provider-execution-gate.ts
|
|
7915
7991
|
var ProviderExecutionGate = class {
|
|
@@ -8115,6 +8191,8 @@ async function runWithConcurrency(items, limit, worker) {
|
|
|
8115
8191
|
await Promise.all(Array.from({ length: cap }, next));
|
|
8116
8192
|
}
|
|
8117
8193
|
var PROVIDER_FANOUT_DEFAULT = 8;
|
|
8194
|
+
var BATCH_INGEST_CHUNK_SIZE = 32;
|
|
8195
|
+
var BATCH_INGEST_PAYLOAD_BYTES = 1024 * 1024;
|
|
8118
8196
|
var SYNC_SLOT_DISPATCH = {
|
|
8119
8197
|
idempotent: false,
|
|
8120
8198
|
mode: ProviderDispatchModes.sync,
|
|
@@ -8256,6 +8334,7 @@ function classifyRunAbortReason(message) {
|
|
|
8256
8334
|
if (/^Run [^ ]+ is not executable\b/.test(message)) return "run_not_executable";
|
|
8257
8335
|
return void 0;
|
|
8258
8336
|
}
|
|
8337
|
+
var FAILURE_STREAK_LOOKBACK = 50;
|
|
8259
8338
|
function classifyProviderErrors(errors) {
|
|
8260
8339
|
return classifyProviderErrorMessages(errors.values());
|
|
8261
8340
|
}
|
|
@@ -8850,21 +8929,28 @@ var JobRunner = class {
|
|
|
8850
8929
|
...executionContext.location ? { location: executionContext.location } : {}
|
|
8851
8930
|
});
|
|
8852
8931
|
} else {
|
|
8853
|
-
|
|
8854
|
-
|
|
8855
|
-
|
|
8856
|
-
|
|
8857
|
-
|
|
8858
|
-
|
|
8859
|
-
|
|
8860
|
-
|
|
8861
|
-
|
|
8862
|
-
|
|
8863
|
-
|
|
8864
|
-
|
|
8865
|
-
|
|
8866
|
-
|
|
8867
|
-
|
|
8932
|
+
const sampling = failureStreakSampling("failed", this.priorFailureStreak(projectId, runId), runId);
|
|
8933
|
+
if (sampling.report) {
|
|
8934
|
+
trackEvent(
|
|
8935
|
+
"run.completed",
|
|
8936
|
+
{
|
|
8937
|
+
...buildRunCompletedProps({
|
|
8938
|
+
status: "failed",
|
|
8939
|
+
providerCount: executionContext.providerCount,
|
|
8940
|
+
providers: executionContext.providers,
|
|
8941
|
+
queryCount: executionContext.queryCount,
|
|
8942
|
+
startTime,
|
|
8943
|
+
trigger: executionContext.trigger,
|
|
8944
|
+
canonicalDomain: executionContext.canonicalDomain,
|
|
8945
|
+
phases,
|
|
8946
|
+
location: executionContext.location
|
|
8947
|
+
}),
|
|
8948
|
+
...describeRunFailure(err, runFailureSite(providerCallStart, providerCallEnd)),
|
|
8949
|
+
...sampling.props
|
|
8950
|
+
},
|
|
8951
|
+
{ errorCode: "INTERNAL" }
|
|
8952
|
+
);
|
|
8953
|
+
}
|
|
8868
8954
|
}
|
|
8869
8955
|
if (this.onRunCompleted) {
|
|
8870
8956
|
this.onRunCompleted(runId, projectId).catch((notifErr) => {
|
|
@@ -8928,21 +9014,28 @@ var JobRunner = class {
|
|
|
8928
9014
|
reportFinalizedRun(input, finalStatus) {
|
|
8929
9015
|
const { runId, projectId, providerErrors, executionContext } = input;
|
|
8930
9016
|
const failureCode = providerErrors.size > 0 ? classifyProviderErrors(providerErrors) : void 0;
|
|
8931
|
-
|
|
8932
|
-
|
|
8933
|
-
|
|
8934
|
-
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
|
|
8941
|
-
|
|
8942
|
-
|
|
8943
|
-
|
|
8944
|
-
|
|
8945
|
-
|
|
9017
|
+
const sampling = failureStreakSampling(finalStatus, this.priorFailureStreak(projectId, runId), runId);
|
|
9018
|
+
if (sampling.report) {
|
|
9019
|
+
trackEvent(
|
|
9020
|
+
"run.completed",
|
|
9021
|
+
{
|
|
9022
|
+
...buildRunCompletedProps({
|
|
9023
|
+
status: finalStatus,
|
|
9024
|
+
providerCount: executionContext.providerCount,
|
|
9025
|
+
providers: executionContext.providers,
|
|
9026
|
+
queryCount: executionContext.queryCount,
|
|
9027
|
+
startTime: input.startTime,
|
|
9028
|
+
trigger: executionContext.trigger,
|
|
9029
|
+
canonicalDomain: executionContext.canonicalDomain,
|
|
9030
|
+
phases: input.phases,
|
|
9031
|
+
location: executionContext.location
|
|
9032
|
+
}),
|
|
9033
|
+
...buildProviderOutcomeProps(executionContext.providers, providerErrors),
|
|
9034
|
+
...sampling.props
|
|
9035
|
+
},
|
|
9036
|
+
failureCode ? { errorCode: failureCode } : void 0
|
|
9037
|
+
);
|
|
9038
|
+
}
|
|
8946
9039
|
if (input.kind === RunKinds["answer-visibility"] && executionContext.trigger !== RunTriggers.probe && input.inserted > 0 && !this.hasPriorActivation(projectId, runId)) {
|
|
8947
9040
|
trackEvent("activation.completed", {
|
|
8948
9041
|
flowVersion: ONBOARDING_FLOW_VERSION,
|
|
@@ -9350,10 +9443,10 @@ var JobRunner = class {
|
|
|
9350
9443
|
/**
|
|
9351
9444
|
* Read an ended batch's results into its run: every line mapped by
|
|
9352
9445
|
* `custom_id` to its slot, parsed by the adapter's own parser and recorded
|
|
9353
|
-
* through
|
|
9446
|
+
* through the shared slot preparation, like a sync answer at the batch tier.
|
|
9354
9447
|
*
|
|
9355
|
-
* Idempotent across crashes.
|
|
9356
|
-
*
|
|
9448
|
+
* Idempotent across crashes. Bounded chunks commit each snapshot together
|
|
9449
|
+
* with its outcome; handled lines are skipped on the next pass. The insert does
|
|
9357
9450
|
* nothing on a slot that is already answered; the reservation of unbilled
|
|
9358
9451
|
* lines is released under `quota_released`, in the same transaction that
|
|
9359
9452
|
* marks the batch `ingested`. Answers are scored against the project's
|
|
@@ -9375,30 +9468,74 @@ var JobRunner = class {
|
|
|
9375
9468
|
const requests = new Map(this.db.select().from(providerBatchRequests).where(eq3(providerBatchRequests.batchId, batch.id)).all().map((row) => [row.id, row]));
|
|
9376
9469
|
const ctx = { ...this.buildRunRecordingContext(batch.runId, batch.projectId), onInserted: () => {
|
|
9377
9470
|
} };
|
|
9471
|
+
const pending = [];
|
|
9472
|
+
let pendingBytes = 0;
|
|
9473
|
+
const queued = /* @__PURE__ */ new Set();
|
|
9474
|
+
const flush = () => {
|
|
9475
|
+
if (pending.length === 0) return true;
|
|
9476
|
+
const chunk = pending.splice(0);
|
|
9477
|
+
pendingBytes = 0;
|
|
9478
|
+
const settled = this.persistBatchLines(batch, chunk);
|
|
9479
|
+
if (!settled) return false;
|
|
9480
|
+
for (const result2 of settled) {
|
|
9481
|
+
const request = requests.get(result2.line.requestId);
|
|
9482
|
+
requests.set(request.id, { ...request, outcome: result2.outcome, error: result2.line.error });
|
|
9483
|
+
result2.line.slot?.report(result2.inserted);
|
|
9484
|
+
result2.line.report?.();
|
|
9485
|
+
}
|
|
9486
|
+
queued.clear();
|
|
9487
|
+
return true;
|
|
9488
|
+
};
|
|
9378
9489
|
try {
|
|
9490
|
+
let linesSinceYield = 0;
|
|
9379
9491
|
for await (const line of capability.results(batch.providerBatchId, registered.config)) {
|
|
9492
|
+
if (linesSinceYield === BATCH_INGEST_CHUNK_SIZE) {
|
|
9493
|
+
await yieldToEventLoop();
|
|
9494
|
+
linesSinceYield = 0;
|
|
9495
|
+
}
|
|
9496
|
+
linesSinceYield++;
|
|
9380
9497
|
const request = requests.get(line.customId);
|
|
9381
9498
|
if (!request) {
|
|
9382
9499
|
log2.warn("batch.unknown-line", { batchId: batch.id, provider: batch.provider, customId: line.customId });
|
|
9383
9500
|
continue;
|
|
9384
9501
|
}
|
|
9385
|
-
if (request.outcome !== null) continue;
|
|
9386
|
-
if (this.db.select({ status: providerBatches.status }).from(providerBatches).where(eq3(providerBatches.id, batch.id)).get()?.status !== ProviderBatchStatuses.ended) {
|
|
9387
|
-
|
|
9502
|
+
if (request.outcome !== null || queued.has(request.id)) continue;
|
|
9503
|
+
if (this.db.select({ status: providerBatches.status }).from(providerBatches).where(eq3(providerBatches.id, batch.id)).get()?.status !== ProviderBatchStatuses.ended) return { kind: "cancelled" };
|
|
9504
|
+
const prepared = await this.prepareBatchLine(ctx, registered, batch, request, line);
|
|
9505
|
+
const payloadBytes = Buffer.byteLength(prepared.slot?.snapshot.rawResponse ?? "") + Buffer.byteLength(prepared.slot?.snapshot.answerText ?? "") + Buffer.byteLength(prepared.error ?? "");
|
|
9506
|
+
if (pending.length > 0 && pendingBytes + payloadBytes > BATCH_INGEST_PAYLOAD_BYTES) {
|
|
9507
|
+
if (!flush()) return { kind: "cancelled" };
|
|
9508
|
+
await yieldToEventLoop();
|
|
9509
|
+
linesSinceYield = 0;
|
|
9510
|
+
}
|
|
9511
|
+
pending.push(prepared);
|
|
9512
|
+
pendingBytes += payloadBytes;
|
|
9513
|
+
queued.add(request.id);
|
|
9514
|
+
if (pending.length === BATCH_INGEST_CHUNK_SIZE || pendingBytes >= BATCH_INGEST_PAYLOAD_BYTES) {
|
|
9515
|
+
if (!flush()) return { kind: "cancelled" };
|
|
9516
|
+
await yieldToEventLoop();
|
|
9517
|
+
linesSinceYield = 0;
|
|
9388
9518
|
}
|
|
9389
|
-
const settled = await this.ingestBatchLine(ctx, registered, batch, request, line);
|
|
9390
|
-
this.db.update(providerBatchRequests).set({ outcome: settled.outcome, error: settled.error }).where(and3(eq3(providerBatchRequests.id, request.id), isNull(providerBatchRequests.outcome))).run();
|
|
9391
|
-
requests.set(request.id, { ...request, outcome: settled.outcome });
|
|
9392
9519
|
}
|
|
9520
|
+
if (!flush()) return { kind: "cancelled" };
|
|
9393
9521
|
} catch (err) {
|
|
9394
|
-
|
|
9522
|
+
let failure2 = err;
|
|
9523
|
+
if (!(failure2 instanceof RunCancelledError)) {
|
|
9524
|
+
try {
|
|
9525
|
+
if (!flush()) return { kind: "cancelled" };
|
|
9526
|
+
} catch (flushError) {
|
|
9527
|
+
failure2 = flushError;
|
|
9528
|
+
}
|
|
9529
|
+
}
|
|
9530
|
+
if (!(failure2 instanceof RunCancelledError)) throw failure2;
|
|
9395
9531
|
this.markBatchesCancelled([batch.id], "Cancelled because its run is no longer running.");
|
|
9396
9532
|
return { kind: "cancelled" };
|
|
9397
9533
|
}
|
|
9398
9534
|
return this.completeBatchIngest(batch);
|
|
9399
9535
|
}
|
|
9400
|
-
async
|
|
9401
|
-
|
|
9536
|
+
async prepareBatchLine(ctx, registered, batch, request, line) {
|
|
9537
|
+
const base = { requestId: request.id, slot: null };
|
|
9538
|
+
if (line.type !== "succeeded") return { ...base, outcome: unansweredOutcome(line.type), error: line.error };
|
|
9402
9539
|
const { adapter } = registered;
|
|
9403
9540
|
if (!adapter.parseTrackedQueryResponse) throw new Error(`Provider ${adapter.name} cannot read a batch answer`);
|
|
9404
9541
|
let raw;
|
|
@@ -9406,34 +9543,67 @@ var JobRunner = class {
|
|
|
9406
9543
|
raw = adapter.parseTrackedQueryResponse(line.body, resolveProviderModel(adapter.name, request.requestedModel));
|
|
9407
9544
|
} catch (err) {
|
|
9408
9545
|
const error = describeError(err);
|
|
9409
|
-
|
|
9410
|
-
|
|
9546
|
+
return {
|
|
9547
|
+
...base,
|
|
9548
|
+
outcome: ProviderBatchRequestOutcomes.parse_failed,
|
|
9549
|
+
error,
|
|
9550
|
+
report: () => log2.warn("batch.parse-failed", { runId: batch.runId, batchId: batch.id, provider: batch.provider, executionId: request.executionId, error })
|
|
9551
|
+
};
|
|
9411
9552
|
}
|
|
9412
|
-
const
|
|
9553
|
+
const slot = await this.prepareSlot(ctx, registered, {
|
|
9413
9554
|
executionId: request.executionId,
|
|
9414
9555
|
queryText: request.queryText,
|
|
9415
9556
|
context: request.requestedContext,
|
|
9416
9557
|
queryId: request.queryId,
|
|
9417
9558
|
requestedModel: request.requestedModel
|
|
9418
|
-
}
|
|
9419
|
-
const written = await this.recordSlot(ctx, registered, unit, raw, {
|
|
9559
|
+
}, raw, {
|
|
9420
9560
|
idempotent: true,
|
|
9421
9561
|
mode: ProviderDispatchModes.batch,
|
|
9422
9562
|
providerBatchId: batch.id,
|
|
9423
9563
|
pricingTier: PricingTiers.batch
|
|
9424
9564
|
});
|
|
9425
|
-
|
|
9426
|
-
|
|
9427
|
-
|
|
9428
|
-
|
|
9429
|
-
|
|
9430
|
-
|
|
9431
|
-
|
|
9565
|
+
return { requestId: request.id, slot, outcome: ProviderBatchRequestOutcomes.recorded, error: null };
|
|
9566
|
+
}
|
|
9567
|
+
/** Snapshots and their replay ledger commit together, with no callbacks or awaits. */
|
|
9568
|
+
persistBatchLines(batch, lines) {
|
|
9569
|
+
return this.db.transaction((tx) => {
|
|
9570
|
+
if (tx.select({ status: providerBatches.status }).from(providerBatches).where(eq3(providerBatches.id, batch.id)).get()?.status !== ProviderBatchStatuses.ended) return null;
|
|
9571
|
+
if (tx.select({ status: runs.status }).from(runs).where(eq3(runs.id, batch.runId)).get()?.status !== RunStatuses.running) throw new RunCancelledError(batch.runId);
|
|
9572
|
+
const current = new Map(tx.select({
|
|
9573
|
+
id: providerBatchRequests.id,
|
|
9574
|
+
outcome: providerBatchRequests.outcome,
|
|
9575
|
+
queryId: providerBatchRequests.queryId,
|
|
9576
|
+
executionId: providerBatchRequests.executionId
|
|
9577
|
+
}).from(providerBatchRequests).where(inArray(providerBatchRequests.id, lines.map((line) => line.requestId))).all().map((row) => [row.id, row]));
|
|
9578
|
+
const settled = [];
|
|
9579
|
+
for (const line of lines) {
|
|
9580
|
+
const request = current.get(line.requestId);
|
|
9581
|
+
if (!request || request.outcome !== null) continue;
|
|
9582
|
+
current.delete(line.requestId);
|
|
9583
|
+
let outcome = line.outcome;
|
|
9584
|
+
let inserted = false;
|
|
9585
|
+
if (line.slot) {
|
|
9586
|
+
const snapshot = { ...line.slot.snapshot, queryId: request.queryId };
|
|
9587
|
+
inserted = tx.insert(querySnapshots).values(snapshot).onConflictDoNothing().run().changes > 0;
|
|
9588
|
+
if (!inserted) {
|
|
9589
|
+
const existing = tx.select({ providerBatchId: querySnapshots.providerBatchId }).from(querySnapshots).where(and3(
|
|
9590
|
+
eq3(querySnapshots.runId, batch.runId),
|
|
9591
|
+
eq3(querySnapshots.measurementExecutionId, request.executionId),
|
|
9592
|
+
eq3(querySnapshots.provider, batch.provider)
|
|
9593
|
+
)).get();
|
|
9594
|
+
outcome = existing?.providerBatchId === batch.id ? ProviderBatchRequestOutcomes.recorded : ProviderBatchRequestOutcomes.duplicate;
|
|
9595
|
+
}
|
|
9596
|
+
}
|
|
9597
|
+
tx.update(providerBatchRequests).set({ outcome, error: line.error }).where(and3(eq3(providerBatchRequests.id, request.id), isNull(providerBatchRequests.outcome))).run();
|
|
9598
|
+
settled.push({ line, outcome, inserted });
|
|
9599
|
+
}
|
|
9600
|
+
return settled;
|
|
9601
|
+
}, { behavior: "immediate" });
|
|
9432
9602
|
}
|
|
9433
9603
|
/** Close an ingest: counts, the unbilled lines' quota, and `ingested`, in one transaction. */
|
|
9434
9604
|
completeBatchIngest(batch) {
|
|
9435
9605
|
const at = (/* @__PURE__ */ new Date()).toISOString();
|
|
9436
|
-
|
|
9606
|
+
const result2 = this.db.transaction((tx) => {
|
|
9437
9607
|
const txDb = tx;
|
|
9438
9608
|
const current = txDb.select().from(providerBatches).where(eq3(providerBatches.id, batch.id)).get();
|
|
9439
9609
|
if (current?.status !== ProviderBatchStatuses.ended) return { kind: "skipped" };
|
|
@@ -9449,9 +9619,19 @@ var JobRunner = class {
|
|
|
9449
9619
|
updatedAt: at
|
|
9450
9620
|
}).where(and3(eq3(providerBatches.id, batch.id), eq3(providerBatches.status, ProviderBatchStatuses.ended))).run();
|
|
9451
9621
|
const { recordedCount: recorded, notRecorded } = tally;
|
|
9452
|
-
log2.info("batch.ingested", { runId: batch.runId, batchId: batch.id, provider: batch.provider, recorded, notRecorded, released });
|
|
9453
9622
|
return { kind: "ingested", recorded, notRecorded, released };
|
|
9454
9623
|
});
|
|
9624
|
+
if (result2.kind === "ingested") {
|
|
9625
|
+
log2.info("batch.ingested", {
|
|
9626
|
+
runId: batch.runId,
|
|
9627
|
+
batchId: batch.id,
|
|
9628
|
+
provider: batch.provider,
|
|
9629
|
+
recorded: result2.recorded,
|
|
9630
|
+
notRecorded: result2.notRecorded,
|
|
9631
|
+
released: result2.released
|
|
9632
|
+
});
|
|
9633
|
+
}
|
|
9634
|
+
return result2;
|
|
9455
9635
|
}
|
|
9456
9636
|
/**
|
|
9457
9637
|
* Give up on an ended batch whose results cannot be read in time. It
|
|
@@ -9742,6 +9922,15 @@ var JobRunner = class {
|
|
|
9742
9922
|
* writes nothing and resolves false.
|
|
9743
9923
|
*/
|
|
9744
9924
|
async recordSlot(ctx, registeredProvider, unit, raw, dispatch) {
|
|
9925
|
+
const prepared = await this.prepareSlot(ctx, registeredProvider, unit, raw, dispatch);
|
|
9926
|
+
this.throwIfRunCancelled(ctx.runId);
|
|
9927
|
+
const insert = this.db.insert(querySnapshots).values(prepared.snapshot);
|
|
9928
|
+
const written = ctx.fill || dispatch.idempotent ? insert.onConflictDoNothing().run() : insert.run();
|
|
9929
|
+
prepared.report(written.changes > 0);
|
|
9930
|
+
return written.changes > 0;
|
|
9931
|
+
}
|
|
9932
|
+
/** Resolve network/file evidence and derived fields before opening a write transaction. */
|
|
9933
|
+
async prepareSlot(ctx, registeredProvider, unit, raw, dispatch) {
|
|
9745
9934
|
const { adapter } = registeredProvider;
|
|
9746
9935
|
const providerName = adapter.name;
|
|
9747
9936
|
const { domains: competitorDomains, aliases: competitorAliases } = ctx.competitorsFor(unit.executionId);
|
|
@@ -9798,7 +9987,7 @@ var JobRunner = class {
|
|
|
9798
9987
|
fs4.renameSync(raw.screenshotPath, destPath);
|
|
9799
9988
|
screenshotRelPath = `${ctx.runId}/${snapshotId}.png`;
|
|
9800
9989
|
}
|
|
9801
|
-
const
|
|
9990
|
+
const snapshot = {
|
|
9802
9991
|
id: snapshotId,
|
|
9803
9992
|
runId: ctx.runId,
|
|
9804
9993
|
queryId: unit.queryId,
|
|
@@ -9850,20 +10039,23 @@ var JobRunner = class {
|
|
|
9850
10039
|
overrides: registeredProvider.config.pricing
|
|
9851
10040
|
}),
|
|
9852
10041
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9853
|
-
}
|
|
9854
|
-
|
|
9855
|
-
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
|
|
9859
|
-
|
|
9860
|
-
|
|
9861
|
-
|
|
9862
|
-
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
10042
|
+
};
|
|
10043
|
+
return {
|
|
10044
|
+
snapshot,
|
|
10045
|
+
report: (inserted) => {
|
|
10046
|
+
if (inserted) ctx.onInserted();
|
|
10047
|
+
ctx.fill?.onOutcome(providerName, true);
|
|
10048
|
+
log2.info("query.citation", {
|
|
10049
|
+
runId: ctx.runId,
|
|
10050
|
+
provider: providerName,
|
|
10051
|
+
query: unit.queryText,
|
|
10052
|
+
executionId: unit.executionId,
|
|
10053
|
+
location: requestedContext?.label ?? null,
|
|
10054
|
+
citationState,
|
|
10055
|
+
answerMentioned
|
|
10056
|
+
});
|
|
10057
|
+
}
|
|
10058
|
+
};
|
|
9867
10059
|
}
|
|
9868
10060
|
/**
|
|
9869
10061
|
* Rebuild a run's recording identity from the database, for a path that
|
|
@@ -9906,6 +10098,36 @@ var JobRunner = class {
|
|
|
9906
10098
|
}
|
|
9907
10099
|
providerReservations.clear();
|
|
9908
10100
|
}
|
|
10101
|
+
/**
|
|
10102
|
+
* How many runs of the same kind, created before this one, failed or came
|
|
10103
|
+
* back partial in a row. Telemetry only: it feeds `failureStreak` and the
|
|
10104
|
+
* sampling of runs stuck failing the same way. Probes do not count, other run
|
|
10105
|
+
* kinds (syncs, audits) neither break nor extend a streak, and a run aborted
|
|
10106
|
+
* before any provider was called (`run.aborted`, never a failed
|
|
10107
|
+
* `run.completed`) is skipped, so every failure counted was also reported.
|
|
10108
|
+
*/
|
|
10109
|
+
priorFailureStreak(projectId, currentRunId) {
|
|
10110
|
+
try {
|
|
10111
|
+
const current = this.db.select({ kind: runs.kind, createdAt: runs.createdAt }).from(runs).where(eq3(runs.id, currentRunId)).get();
|
|
10112
|
+
if (!current) return 0;
|
|
10113
|
+
const recent = this.db.select({ status: runs.status, error: runs.error }).from(runs).where(and3(
|
|
10114
|
+
eq3(runs.projectId, projectId),
|
|
10115
|
+
eq3(runs.kind, current.kind),
|
|
10116
|
+
lt(runs.createdAt, current.createdAt),
|
|
10117
|
+
ne(runs.trigger, RunTriggers.probe),
|
|
10118
|
+
inArray(runs.status, ["completed", "partial", "failed"])
|
|
10119
|
+
)).orderBy(desc(runs.createdAt)).limit(FAILURE_STREAK_LOOKBACK).all();
|
|
10120
|
+
let streak = 0;
|
|
10121
|
+
for (const row of recent) {
|
|
10122
|
+
if (row.status === "completed") break;
|
|
10123
|
+
if (row.status === "failed" && row.error && classifyRunAbortReason(row.error)) continue;
|
|
10124
|
+
streak++;
|
|
10125
|
+
}
|
|
10126
|
+
return streak;
|
|
10127
|
+
} catch {
|
|
10128
|
+
return 0;
|
|
10129
|
+
}
|
|
10130
|
+
}
|
|
9909
10131
|
hasPriorActivation(projectId, currentRunId) {
|
|
9910
10132
|
return this.db.select({ id: querySnapshots.id }).from(querySnapshots).innerJoin(runs, eq3(querySnapshots.runId, runs.id)).where(and3(
|
|
9911
10133
|
eq3(runs.projectId, projectId),
|
|
@@ -9997,7 +10219,7 @@ import { eq as eq5, and as and5, sql as sql3 } from "drizzle-orm";
|
|
|
9997
10219
|
|
|
9998
10220
|
// src/gsc-coverage-snapshot.ts
|
|
9999
10221
|
import crypto7 from "crypto";
|
|
10000
|
-
import { and as and4, desc, eq as eq4, gte } from "drizzle-orm";
|
|
10222
|
+
import { and as and4, desc as desc2, eq as eq4, gte } from "drizzle-orm";
|
|
10001
10223
|
var COVERAGE_IMPRESSION_WINDOW_DAYS = 30;
|
|
10002
10224
|
function pageKey(raw) {
|
|
10003
10225
|
if (!raw) return null;
|
|
@@ -10021,9 +10243,9 @@ function writeCoverageSnapshot(db, projectId, runId, opts = {}) {
|
|
|
10021
10243
|
eq4(siteCrawlSnapshots.projectId, projectId),
|
|
10022
10244
|
eq4(siteCrawlSnapshots.complete, true),
|
|
10023
10245
|
eq4(runs.status, "completed")
|
|
10024
|
-
)).orderBy(
|
|
10246
|
+
)).orderBy(desc2(siteCrawlSnapshots.createdAt)).limit(1).get();
|
|
10025
10247
|
const hasCrawlHistory = Boolean(db.select({ id: siteCrawlSnapshots.id }).from(siteCrawlSnapshots).where(eq4(siteCrawlSnapshots.projectId, projectId)).limit(1).get());
|
|
10026
|
-
const latestAudit = db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).where(eq4(siteAuditSnapshots.projectId, projectId)).orderBy(
|
|
10248
|
+
const latestAudit = db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).where(eq4(siteAuditSnapshots.projectId, projectId)).orderBy(desc2(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
10027
10249
|
const sitemapUrls = latestCrawl?.attemptId ? db.select({ url: siteCrawlPages.url }).from(siteCrawlPages).where(and4(
|
|
10028
10250
|
eq4(siteCrawlPages.projectId, projectId),
|
|
10029
10251
|
eq4(siteCrawlPages.runId, latestCrawl.runId),
|
|
@@ -10244,11 +10466,11 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
10244
10466
|
|
|
10245
10467
|
// src/gbp-sync.ts
|
|
10246
10468
|
import crypto10 from "crypto";
|
|
10247
|
-
import { eq as eq7, and as and7, desc as
|
|
10469
|
+
import { eq as eq7, and as and7, desc as desc4, inArray as inArray3, lt as lt2 } from "drizzle-orm";
|
|
10248
10470
|
|
|
10249
10471
|
// src/gbp-reviews.ts
|
|
10250
10472
|
import crypto9 from "crypto";
|
|
10251
|
-
import { and as and6, desc as
|
|
10473
|
+
import { and as and6, desc as desc3, eq as eq6, inArray as inArray2, sql as sql4 } from "drizzle-orm";
|
|
10252
10474
|
var log4 = createLogger("GbpReviews");
|
|
10253
10475
|
var MS_PER_DAY = 864e5;
|
|
10254
10476
|
var REVIEW_ALERT_MAX_AGE_DAYS = 30;
|
|
@@ -10280,7 +10502,7 @@ function placesReviewsDue(db, projectId, locationName, now) {
|
|
|
10280
10502
|
eq6(gbpReviewRatings.projectId, projectId),
|
|
10281
10503
|
eq6(gbpReviewRatings.locationName, locationName),
|
|
10282
10504
|
eq6(gbpReviewRatings.origin, "places")
|
|
10283
|
-
)).orderBy(
|
|
10505
|
+
)).orderBy(desc3(gbpReviewRatings.observedAt)).limit(1).get();
|
|
10284
10506
|
if (!latest) return true;
|
|
10285
10507
|
return now.getTime() - Date.parse(latest.observedAt) >= PLACES_REVIEW_MIN_INTERVAL_HOURS * 36e5;
|
|
10286
10508
|
}
|
|
@@ -10361,7 +10583,7 @@ function persistReviewObservation(tx, projectId, runId, obs, ctx) {
|
|
|
10361
10583
|
eq6(gbpReviewRatings.projectId, projectId),
|
|
10362
10584
|
eq6(gbpReviewRatings.locationName, obs.locationName),
|
|
10363
10585
|
eq6(gbpReviewRatings.origin, obs.origin)
|
|
10364
|
-
)).orderBy(
|
|
10586
|
+
)).orderBy(desc3(gbpReviewRatings.observedAt)).limit(1).get();
|
|
10365
10587
|
const baseline = latestRating === void 0;
|
|
10366
10588
|
let queuedRatingDrop = false;
|
|
10367
10589
|
if (!latestRating) {
|
|
@@ -10567,15 +10789,15 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
10567
10789
|
fetchLocationReviews(db, projectId, loc, reviewContext)
|
|
10568
10790
|
]);
|
|
10569
10791
|
const lodgingHash = lodging ? hashLodging(lodging) : null;
|
|
10570
|
-
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(and7(eq7(gbpLodgingSnapshots.projectId, projectId), eq7(gbpLodgingSnapshots.locationName, loc.locationName))).orderBy(
|
|
10792
|
+
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(and7(eq7(gbpLodgingSnapshots.projectId, projectId), eq7(gbpLodgingSnapshots.locationName, loc.locationName))).orderBy(desc4(gbpLodgingSnapshots.syncedAt)).limit(1).get() : void 0;
|
|
10571
10793
|
const lodgingChanged = lodging !== null && latestLodging?.contentHash !== lodgingHash;
|
|
10572
10794
|
const attributesHash = hashAttributes(attributes);
|
|
10573
|
-
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(and7(eq7(gbpAttributesSnapshots.projectId, projectId), eq7(gbpAttributesSnapshots.locationName, loc.locationName))).orderBy(
|
|
10795
|
+
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(and7(eq7(gbpAttributesSnapshots.projectId, projectId), eq7(gbpAttributesSnapshots.locationName, loc.locationName))).orderBy(desc4(gbpAttributesSnapshots.syncedAt)).limit(1).get();
|
|
10574
10796
|
const attributesChanged = latestAttributes?.contentHash !== attributesHash;
|
|
10575
10797
|
let placeToWrite = null;
|
|
10576
10798
|
let placeToTouch = null;
|
|
10577
10799
|
if (placesTier !== "off" && placesApiKey && lodging !== null && loc.placeId) {
|
|
10578
|
-
const latestPlace = db.select().from(gbpPlaceDetails).where(and7(eq7(gbpPlaceDetails.projectId, projectId), eq7(gbpPlaceDetails.locationName, loc.locationName))).orderBy(
|
|
10800
|
+
const latestPlace = db.select().from(gbpPlaceDetails).where(and7(eq7(gbpPlaceDetails.projectId, projectId), eq7(gbpPlaceDetails.locationName, loc.locationName))).orderBy(desc4(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
10579
10801
|
const ageDays = latestPlace ? (Date.now() - new Date(latestPlace.syncedAt).getTime()) / MS_PER_DAY2 : Infinity;
|
|
10580
10802
|
if (ageDays >= placesRefreshDays) {
|
|
10581
10803
|
try {
|
|
@@ -10645,7 +10867,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
10645
10867
|
tx.delete(gbpKeywordMonthly).where(and7(
|
|
10646
10868
|
eq7(gbpKeywordMonthly.projectId, projectId),
|
|
10647
10869
|
eq7(gbpKeywordMonthly.locationName, loc.locationName),
|
|
10648
|
-
|
|
10870
|
+
lt2(gbpKeywordMonthly.month, keywordRetentionCutoff)
|
|
10649
10871
|
)).run();
|
|
10650
10872
|
}
|
|
10651
10873
|
tx.delete(gbpPlaceActions).where(and7(eq7(gbpPlaceActions.projectId, projectId), eq7(gbpPlaceActions.locationName, loc.locationName))).run();
|
|
@@ -11973,7 +12195,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
11973
12195
|
|
|
11974
12196
|
// src/bing-inspect-sitemap.ts
|
|
11975
12197
|
import crypto14 from "crypto";
|
|
11976
|
-
import { eq as eq10, desc as
|
|
12198
|
+
import { eq as eq10, desc as desc5 } from "drizzle-orm";
|
|
11977
12199
|
var log9 = createLogger("BingInspectSitemap");
|
|
11978
12200
|
var BING_INSPECT_SPACING_MS = 2e3;
|
|
11979
12201
|
function parseBingDate(value) {
|
|
@@ -12137,7 +12359,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
12137
12359
|
`Bing inspection failed for every URL attempted (${errors} of ${targetUrls.length}); coverage was left unchanged rather than re-dated from stored inspections. Last error: ${detail}`
|
|
12138
12360
|
);
|
|
12139
12361
|
}
|
|
12140
|
-
const allInspections = db.select().from(bingUrlInspections).where(eq10(bingUrlInspections.projectId, projectId)).orderBy(
|
|
12362
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq10(bingUrlInspections.projectId, projectId)).orderBy(desc5(bingUrlInspections.inspectedAt)).all();
|
|
12141
12363
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
12142
12364
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
12143
12365
|
for (const row of allInspections) {
|
|
@@ -12203,7 +12425,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
12203
12425
|
|
|
12204
12426
|
// src/coverage-refresh.ts
|
|
12205
12427
|
import crypto15 from "crypto";
|
|
12206
|
-
import { and as and9, desc as
|
|
12428
|
+
import { and as and9, desc as desc6, eq as eq11, inArray as inArray5 } from "drizzle-orm";
|
|
12207
12429
|
var log10 = createLogger("CoverageRefresh");
|
|
12208
12430
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
12209
12431
|
var COVERAGE_REFRESH_MANUAL_MIN_INTERVAL_MS = 2 * 60 * 1e3;
|
|
@@ -12232,7 +12454,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
12232
12454
|
eq11(runs.kind, RunKinds["inspect-sitemap"]),
|
|
12233
12455
|
inArray5(runs.status, IN_FLIGHT_STATUSES)
|
|
12234
12456
|
)
|
|
12235
|
-
).orderBy(
|
|
12457
|
+
).orderBy(desc6(runs.createdAt)).limit(1).get();
|
|
12236
12458
|
if (inFlight2) {
|
|
12237
12459
|
log10.info("skip.in-flight", { projectId });
|
|
12238
12460
|
return null;
|
|
@@ -12244,7 +12466,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
12244
12466
|
eq11(runs.kind, RunKinds["inspect-sitemap"]),
|
|
12245
12467
|
inArray5(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
12246
12468
|
)
|
|
12247
|
-
).orderBy(
|
|
12469
|
+
).orderBy(desc6(runs.createdAt)).limit(1).get();
|
|
12248
12470
|
if (recent) {
|
|
12249
12471
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
12250
12472
|
if (Number.isFinite(ageMs) && ageMs < windowMs) {
|
|
@@ -12469,7 +12691,7 @@ function computeSummary(rows) {
|
|
|
12469
12691
|
// src/backlink-extract.ts
|
|
12470
12692
|
import crypto17 from "crypto";
|
|
12471
12693
|
import fs5 from "fs";
|
|
12472
|
-
import { and as and11, desc as
|
|
12694
|
+
import { and as and11, desc as desc7, eq as eq13 } from "drizzle-orm";
|
|
12473
12695
|
var log12 = createLogger("BacklinkExtract");
|
|
12474
12696
|
function defaultDeps3() {
|
|
12475
12697
|
return {
|
|
@@ -12487,7 +12709,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
12487
12709
|
if (!project) {
|
|
12488
12710
|
throw new Error(`Project not found: ${projectId}`);
|
|
12489
12711
|
}
|
|
12490
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq13(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq13(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(
|
|
12712
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq13(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq13(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc7(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
12491
12713
|
if (!sync) {
|
|
12492
12714
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
12493
12715
|
}
|
|
@@ -13391,10 +13613,13 @@ function deadLinkCheckedCount(edges, pages) {
|
|
|
13391
13613
|
}
|
|
13392
13614
|
function trackSiteAuditOutcome(db, input) {
|
|
13393
13615
|
try {
|
|
13394
|
-
const errorCode = input.status === "cancelled" ? "RUN_CANCELLED" : input.status === "failed" ? "
|
|
13616
|
+
const errorCode = input.status === "cancelled" ? "RUN_CANCELLED" : input.status === "failed" ? "INTERNAL" : void 0;
|
|
13395
13617
|
trackEvent(
|
|
13396
13618
|
"site_audit.completed",
|
|
13397
|
-
|
|
13619
|
+
{
|
|
13620
|
+
...buildSiteAuditCompletedProps(input),
|
|
13621
|
+
...input.status === "failed" ? describeRunFailure(input.error) : {}
|
|
13622
|
+
},
|
|
13398
13623
|
errorCode ? { errorCode } : void 0
|
|
13399
13624
|
);
|
|
13400
13625
|
const pagesAudited = input.crawl?.pagesAudited ?? 0;
|
|
@@ -13961,15 +14186,18 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
13961
14186
|
status: cancelled ? "cancelled" : "failed",
|
|
13962
14187
|
startTime,
|
|
13963
14188
|
trigger,
|
|
13964
|
-
canonicalDomain
|
|
14189
|
+
canonicalDomain,
|
|
14190
|
+
error
|
|
13965
14191
|
});
|
|
13966
14192
|
throw error;
|
|
13967
14193
|
}
|
|
13968
14194
|
}
|
|
13969
14195
|
|
|
13970
14196
|
// src/commands/backfill.ts
|
|
13971
|
-
import {
|
|
14197
|
+
import { setImmediate as yieldToEventLoop2 } from "timers/promises";
|
|
14198
|
+
import { and as and15, asc as asc4, desc as desc8, eq as eq17, gt as gt3, inArray as inArray7, isNull as isNull2, lte as lte2, sql as sql7 } from "drizzle-orm";
|
|
13972
14199
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
14200
|
+
var PATH_REPAIR_PAGE_SIZE = 128;
|
|
13973
14201
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
13974
14202
|
const config = loadConfig();
|
|
13975
14203
|
const db = createClient(config.database);
|
|
@@ -14150,35 +14378,54 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
14150
14378
|
}
|
|
14151
14379
|
}
|
|
14152
14380
|
function backfillNormalizedPaths(db, opts) {
|
|
14153
|
-
|
|
14154
|
-
|
|
14155
|
-
|
|
14156
|
-
|
|
14157
|
-
const
|
|
14158
|
-
|
|
14159
|
-
|
|
14160
|
-
|
|
14161
|
-
|
|
14381
|
+
return repairStoredPaths(db, gaTrafficSnapshots, opts);
|
|
14382
|
+
}
|
|
14383
|
+
async function repairStoredPaths(db, table, opts) {
|
|
14384
|
+
const pageSize = opts?.pageSize ?? PATH_REPAIR_PAGE_SIZE;
|
|
14385
|
+
const projectScope = opts?.projectId ? sql7`+${table.projectId} = ${opts.projectId}` : void 0;
|
|
14386
|
+
const lastId = db.select({ id: table.id }).from(table).where(projectScope).orderBy(desc8(table.id)).limit(1).get()?.id;
|
|
14387
|
+
if (lastId === void 0) return { examined: 0, updated: 0, unchanged: 0 };
|
|
14388
|
+
let cursor;
|
|
14389
|
+
let examined = 0;
|
|
14162
14390
|
let updated = 0;
|
|
14163
|
-
|
|
14164
|
-
|
|
14165
|
-
|
|
14166
|
-
|
|
14167
|
-
|
|
14168
|
-
|
|
14169
|
-
|
|
14170
|
-
|
|
14171
|
-
|
|
14172
|
-
|
|
14173
|
-
|
|
14174
|
-
|
|
14175
|
-
|
|
14176
|
-
|
|
14177
|
-
|
|
14178
|
-
}
|
|
14391
|
+
for (; ; ) {
|
|
14392
|
+
const rows = db.select({
|
|
14393
|
+
id: table.id,
|
|
14394
|
+
projectId: table.projectId,
|
|
14395
|
+
landingPage: table.landingPage,
|
|
14396
|
+
landingPageNormalized: table.landingPageNormalized
|
|
14397
|
+
}).from(table).where(and15(
|
|
14398
|
+
projectScope,
|
|
14399
|
+
lte2(table.id, lastId),
|
|
14400
|
+
cursor === void 0 ? void 0 : gt3(table.id, cursor)
|
|
14401
|
+
)).orderBy(asc4(table.id)).limit(pageSize).all();
|
|
14402
|
+
examined += rows.length;
|
|
14403
|
+
const changes = rows.flatMap((row) => {
|
|
14404
|
+
const next = normalizeUrlPath(row.landingPage);
|
|
14405
|
+
return next === null || row.landingPageNormalized === next ? [] : [{ row, next }];
|
|
14179
14406
|
});
|
|
14407
|
+
if (changes.length > 0) {
|
|
14408
|
+
const written = db.transaction((tx) => {
|
|
14409
|
+
let written2 = 0;
|
|
14410
|
+
for (const { row, next } of changes) {
|
|
14411
|
+
written2 += tx.update(table).set({ landingPageNormalized: next }).where(and15(
|
|
14412
|
+
eq17(table.id, row.id),
|
|
14413
|
+
eq17(table.projectId, row.projectId),
|
|
14414
|
+
eq17(table.landingPage, row.landingPage),
|
|
14415
|
+
row.landingPageNormalized === null ? isNull2(table.landingPageNormalized) : eq17(table.landingPageNormalized, row.landingPageNormalized)
|
|
14416
|
+
)).run().changes;
|
|
14417
|
+
}
|
|
14418
|
+
return written2;
|
|
14419
|
+
});
|
|
14420
|
+
updated += written;
|
|
14421
|
+
if (written < changes.length) opts?.onConflict?.();
|
|
14422
|
+
}
|
|
14423
|
+
if (rows.length < pageSize) break;
|
|
14424
|
+
cursor = rows.at(-1).id;
|
|
14425
|
+
if (cursor === lastId) break;
|
|
14426
|
+
await yieldToEventLoop2();
|
|
14180
14427
|
}
|
|
14181
|
-
return { examined
|
|
14428
|
+
return { examined, updated, unchanged: examined - updated };
|
|
14182
14429
|
}
|
|
14183
14430
|
async function backfillNormalizedPathsCommand(opts) {
|
|
14184
14431
|
const config = loadConfig();
|
|
@@ -14204,7 +14451,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
14204
14451
|
}
|
|
14205
14452
|
projectId = project.id;
|
|
14206
14453
|
}
|
|
14207
|
-
const { examined, updated, unchanged } = backfillNormalizedPaths(db, { projectId });
|
|
14454
|
+
const { examined, updated, unchanged } = await backfillNormalizedPaths(db, { projectId });
|
|
14208
14455
|
const result2 = {
|
|
14209
14456
|
project: projectFilter ?? null,
|
|
14210
14457
|
examined,
|
|
@@ -14222,35 +14469,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
14222
14469
|
console.log(` Unchanged: ${unchanged}`);
|
|
14223
14470
|
}
|
|
14224
14471
|
function backfillAiReferralPaths(db, opts) {
|
|
14225
|
-
|
|
14226
|
-
if (opts?.projectId) {
|
|
14227
|
-
baseConditions.push(eq17(gaAiReferrals.projectId, opts.projectId));
|
|
14228
|
-
}
|
|
14229
|
-
const rows = db.select({
|
|
14230
|
-
id: gaAiReferrals.id,
|
|
14231
|
-
landingPage: gaAiReferrals.landingPage,
|
|
14232
|
-
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
14233
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and15(...baseConditions) : void 0).all();
|
|
14234
|
-
let updated = 0;
|
|
14235
|
-
let unchanged = 0;
|
|
14236
|
-
if (rows.length > 0) {
|
|
14237
|
-
db.transaction((tx) => {
|
|
14238
|
-
for (const row of rows) {
|
|
14239
|
-
const next = normalizeUrlPath(row.landingPage);
|
|
14240
|
-
if (next === null) {
|
|
14241
|
-
unchanged++;
|
|
14242
|
-
continue;
|
|
14243
|
-
}
|
|
14244
|
-
if (row.landingPageNormalized === next) {
|
|
14245
|
-
unchanged++;
|
|
14246
|
-
continue;
|
|
14247
|
-
}
|
|
14248
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq17(gaAiReferrals.id, row.id)).run();
|
|
14249
|
-
updated++;
|
|
14250
|
-
}
|
|
14251
|
-
});
|
|
14252
|
-
}
|
|
14253
|
-
return { examined: rows.length, updated, unchanged };
|
|
14472
|
+
return repairStoredPaths(db, gaAiReferrals, opts);
|
|
14254
14473
|
}
|
|
14255
14474
|
async function backfillAiReferralPathsCommand(opts) {
|
|
14256
14475
|
const config = loadConfig();
|
|
@@ -14276,7 +14495,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
14276
14495
|
}
|
|
14277
14496
|
projectId = project.id;
|
|
14278
14497
|
}
|
|
14279
|
-
const { examined, updated, unchanged } = backfillAiReferralPaths(db, { projectId });
|
|
14498
|
+
const { examined, updated, unchanged } = await backfillAiReferralPaths(db, { projectId });
|
|
14280
14499
|
const result2 = {
|
|
14281
14500
|
project: projectFilter ?? null,
|
|
14282
14501
|
examined,
|
|
@@ -14454,7 +14673,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
14454
14673
|
return result2;
|
|
14455
14674
|
}
|
|
14456
14675
|
async function backfillInsightsCommand(project, opts) {
|
|
14457
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
14676
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-4IONMOFB.js");
|
|
14458
14677
|
const config = loadConfig();
|
|
14459
14678
|
const db = createClient(config.database);
|
|
14460
14679
|
migrate(db);
|
|
@@ -15871,7 +16090,7 @@ async function refreshAllIntegrations(client, projectName, now = /* @__PURE__ */
|
|
|
15871
16090
|
}
|
|
15872
16091
|
|
|
15873
16092
|
// src/notifier.ts
|
|
15874
|
-
import { eq as eq20, desc as
|
|
16093
|
+
import { eq as eq20, desc as desc9, and as and18, inArray as inArray10, or } from "drizzle-orm";
|
|
15875
16094
|
import crypto21 from "crypto";
|
|
15876
16095
|
var log19 = createLogger("Notifier");
|
|
15877
16096
|
var MATERIAL_MAGNITUDE_DELTA = 20;
|
|
@@ -16412,7 +16631,7 @@ var Notifier = class {
|
|
|
16412
16631
|
eq20(runs.kind, thisRun.kind),
|
|
16413
16632
|
or(eq20(runs.status, "completed"), eq20(runs.status, "partial"))
|
|
16414
16633
|
)
|
|
16415
|
-
).orderBy(
|
|
16634
|
+
).orderBy(desc9(runs.createdAt), desc9(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
16416
16635
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
16417
16636
|
const currentGroupIdx = groups.findIndex((g) => g[0]?.createdAt === thisRun.createdAt);
|
|
16418
16637
|
if (currentGroupIdx < 0) return [];
|
|
@@ -16733,12 +16952,17 @@ var PROVIDER_MODELS = {
|
|
|
16733
16952
|
[LlmCapabilities.classify]: "gpt-5-nano"
|
|
16734
16953
|
},
|
|
16735
16954
|
[AgentProviderIds.gemini]: {
|
|
16736
|
-
// Gemini's
|
|
16955
|
+
// Gemini's Flash is already cheap + capable; Flash-Lite is the
|
|
16737
16956
|
// dedicated micro tier for classify. Analyze stays on flash because
|
|
16738
16957
|
// flash-lite drops too much quality for structured synthesis.
|
|
16739
|
-
|
|
16740
|
-
|
|
16741
|
-
|
|
16958
|
+
// Exact IDs, never Google's `-latest` aliases: pi-ai derives request
|
|
16959
|
+
// options (thinking level) from the catalog entry for the ID it is given,
|
|
16960
|
+
// and an alias entry can lag the model it serves. `gemini-flash-latest`
|
|
16961
|
+
// allows MINIMAL thinking while the 3.8 Flash it resolves to rejects it,
|
|
16962
|
+
// so every Aero turn failed. Sweeps can use the alias; Aero cannot.
|
|
16963
|
+
[LlmCapabilities.agent]: "gemini-3.8-flash",
|
|
16964
|
+
[LlmCapabilities.analyze]: "gemini-3.8-flash",
|
|
16965
|
+
[LlmCapabilities.classify]: "gemini-3.5-flash-lite"
|
|
16742
16966
|
},
|
|
16743
16967
|
[AgentProviderIds.zai]: {
|
|
16744
16968
|
// GLM lineage: 5.2 replaced 5.1, which pi-ai 0.87 dropped from its
|
|
@@ -18631,7 +18855,7 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
18631
18855
|
|
|
18632
18856
|
// src/agent/memory-store.ts
|
|
18633
18857
|
import crypto24 from "crypto";
|
|
18634
|
-
import { and as and19, desc as
|
|
18858
|
+
import { and as and19, desc as desc10, eq as eq22, like, sql as sql10, or as or2 } from "drizzle-orm";
|
|
18635
18859
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
18636
18860
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
18637
18861
|
function rowToDto(row) {
|
|
@@ -18645,7 +18869,7 @@ function rowToDto(row) {
|
|
|
18645
18869
|
};
|
|
18646
18870
|
}
|
|
18647
18871
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
18648
|
-
const query = db.select().from(agentMemory).where(eq22(agentMemory.projectId, projectId)).orderBy(
|
|
18872
|
+
const query = db.select().from(agentMemory).where(eq22(agentMemory.projectId, projectId)).orderBy(desc10(agentMemory.updatedAt));
|
|
18649
18873
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
18650
18874
|
return rows.map(rowToDto);
|
|
18651
18875
|
}
|
|
@@ -18691,7 +18915,7 @@ function loadRecentForHydrate(db, projectId, limit, sessionId) {
|
|
|
18691
18915
|
return db.select().from(agentMemory).where(and19(
|
|
18692
18916
|
eq22(agentMemory.projectId, projectId),
|
|
18693
18917
|
or2(sql10`substr(${agentMemory.key}, 1, ${COMPACTION_KEY_PREFIX.length}) <> ${COMPACTION_KEY_PREFIX}`, sessionId ? sql10`substr(${agentMemory.key}, 1, ${sessionPrefix.length}) = ${sessionPrefix}` : void 0)
|
|
18694
|
-
)).orderBy(
|
|
18918
|
+
)).orderBy(desc10(agentMemory.updatedAt)).limit(limit).all().map(rowToDto);
|
|
18695
18919
|
}
|
|
18696
18920
|
function writeCompactionNote(db, args) {
|
|
18697
18921
|
if (Buffer.byteLength(args.summary, "utf8") > AGENT_MEMORY_VALUE_MAX_BYTES) {
|
|
@@ -18719,7 +18943,7 @@ function writeCompactionNote(db, args) {
|
|
|
18719
18943
|
eq22(agentMemory.projectId, args.projectId),
|
|
18720
18944
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
18721
18945
|
)
|
|
18722
|
-
).orderBy(
|
|
18946
|
+
).orderBy(desc10(agentMemory.updatedAt)).all();
|
|
18723
18947
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
18724
18948
|
if (stale.length > 0) {
|
|
18725
18949
|
tx.delete(agentMemory).where(sql10`${agentMemory.id} IN (${sql10.join(stale.map((s) => sql10`${s}`), sql10`, `)})`).run();
|