@analitikisler/ai-tag-debugger 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +174 -47
- package/README.tr.md +305 -178
- package/package.json +5 -2
- package/src/agent.js +1 -1
- package/src/audit.js +27 -13
- package/src/checks.js +123 -1
- package/src/cli.js +116 -71
- package/src/discover.js +148 -0
- package/src/help.js +104 -0
- package/src/i18n.js +22 -0
- package/src/journeys.js +87 -8
- package/src/mcp.js +52 -11
- package/src/plan-html.js +76 -0
- package/src/plan.js +14 -0
- package/src/providers.js +1 -1
- package/src/replay.js +47 -0
- package/src/report.js +34 -10
- package/src/sheet.js +321 -0
- package/src/workspace.js +77 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@analitikisler/ai-tag-debugger",
|
|
3
3
|
"mcpName": "io.github.analitikisler/ai-tag-debugger",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.10.0",
|
|
5
5
|
"description": "Open-source AI agent that QA-tests your website's analytics, ad and consent tracking (GA4, GTM, Google Ads, Meta, TikTok). Bring your own model (Claude, OpenAI, Gemini) or use it from your chat app over MCP.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"setup": "node scripts/setup.js",
|
|
19
19
|
"test": "node --test test/*.test.js",
|
|
20
20
|
"demo:serve": "node examples/serve.js",
|
|
21
|
-
"demo": "node src/cli.js run --config examples/demo.config.json --scripted --no-ai
|
|
21
|
+
"demo": "node src/cli.js run --config examples/demo.config.json --scripted --no-ai"
|
|
22
22
|
},
|
|
23
23
|
"engines": {
|
|
24
24
|
"node": ">=20"
|
|
@@ -57,5 +57,8 @@
|
|
|
57
57
|
"openai": "^7.30.1",
|
|
58
58
|
"playwright": "1.56.1",
|
|
59
59
|
"zod": "^4.6.5"
|
|
60
|
+
},
|
|
61
|
+
"optionalDependencies": {
|
|
62
|
+
"fflate": "^0.8.3"
|
|
60
63
|
}
|
|
61
64
|
}
|
package/src/agent.js
CHANGED
|
@@ -95,7 +95,7 @@ const TIMEOUTS = { model: 180_000, action: 120_000, page: 60_000 };
|
|
|
95
95
|
|
|
96
96
|
const LAST_STEP_WAIT_MS = 3000;
|
|
97
97
|
|
|
98
|
-
class StepTimeout extends Error {}
|
|
98
|
+
export class StepTimeout extends Error {}
|
|
99
99
|
|
|
100
100
|
/** The promise's value, or a StepTimeout with this message once ms have passed. */
|
|
101
101
|
function withTimeout(promise, ms, message) {
|
package/src/audit.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
// A full audit from a config file: run every journey, check, explain, write the reports.
|
|
2
2
|
// Shared by the CLI and the MCP server's run_audit tool.
|
|
3
|
+
import { newRunDir } from "./workspace.js";
|
|
3
4
|
import { readFile, mkdir, rm, writeFile } from "node:fs/promises";
|
|
4
5
|
import path from "node:path";
|
|
5
6
|
import { failedRun, runJourney } from "./agent.js";
|
|
6
7
|
import { VIEWPORTS } from "./browser.js";
|
|
7
|
-
import { checkRun, groupFindings, summarize } from "./checks.js";
|
|
8
|
+
import { checkRun, groupFindings, planCheck, summarize } from "./checks.js";
|
|
8
9
|
import { explainFindings, renderCsv, renderHtml, renderMarkdown } from "./report.js";
|
|
9
10
|
import { LANGUAGES, messages } from "./i18n.js";
|
|
10
11
|
import { createProvider } from "./providers.js";
|
|
11
|
-
import { filterFindings, planRun } from "./journeys.js";
|
|
12
|
+
import { filterFindings, JOURNEY_PRESETS, planRun } from "./journeys.js";
|
|
12
13
|
import { resolveProviders } from "./parsers.js";
|
|
13
14
|
import { SAVED_SITE_FILE, writeSavedSite } from "./replay.js";
|
|
14
15
|
|
|
@@ -32,30 +33,33 @@ export async function loadConfig(configFile) {
|
|
|
32
33
|
return { config, plan, gtmContainer };
|
|
33
34
|
}
|
|
34
35
|
|
|
35
|
-
export const defaultOutDir = () => path.resolve("reports", new Date().toISOString().replace(/[:.]/g, "-"));
|
|
36
36
|
|
|
37
|
-
/** Writes report.md, report.html, timeline.csv and capture.json. */
|
|
38
|
-
export async function writeReports({ outDir, config, plan, runs, findings, explained, lang, branding = config.branding !== false }) {
|
|
37
|
+
/** Writes report.md, report.html, timeline.csv and capture.json, plus plan-check.json with --against. */
|
|
38
|
+
export async function writeReports({ outDir, config, plan, runs, findings, explained, lang, against = null, branding = config.branding !== false }) {
|
|
39
39
|
await mkdir(outDir, { recursive: true });
|
|
40
40
|
const title = messages(lang).ui.title(new URL(config.site).hostname);
|
|
41
41
|
const generatedAt = new Date().toISOString().slice(0, 16).replace("T", " ") + " UTC";
|
|
42
|
-
const
|
|
42
|
+
const check = against ? { plan_id: against.id ?? null, ...planCheck(plan, runs, findings) } : null;
|
|
43
|
+
const report = { title, generatedAt, findings, explained, runs, plan, lang, branding, against: against?.label ?? null, planCheck: check };
|
|
43
44
|
await writeFile(path.join(outDir, "report.md"), renderMarkdown(report));
|
|
44
45
|
await writeFile(path.join(outDir, "report.html"), renderHtml(report));
|
|
45
46
|
await writeFile(path.join(outDir, "timeline.csv"), renderCsv(report));
|
|
46
|
-
await writeFile(path.join(outDir, "
|
|
47
|
+
if (check) await writeFile(path.join(outDir, "plan-check.json"), JSON.stringify(check, null, 2));
|
|
48
|
+
await writeFile(path.join(outDir, "capture.json"), JSON.stringify({ config: { ...config, plan }, ...(against ? { against: { plan_id: against.id, file: against.file } } : {}), runs, findings, explained }, null, 2));
|
|
47
49
|
return path.join(outDir, "report.html");
|
|
48
50
|
}
|
|
49
51
|
|
|
50
52
|
/**
|
|
51
53
|
* @param {{ config: any, plan?: any, gtmContainer?: any, journeys?: string[], goals?: string[], events?: string[], viewports?: string[],
|
|
52
54
|
* scripted?: boolean, ai?: boolean, provider?: string, model?: string, baseUrl?: string, maxSteps?: number, lang?: string, headed?: boolean,
|
|
53
|
-
* outDir?: string, branding?: boolean, providers?: string[], saveSite?: string, replaySite?: string, log?: (m: string) => void }} opts
|
|
55
|
+
* outDir?: string, dataDir?: string, branding?: boolean, providers?: string[], saveSite?: string, replaySite?: string, log?: (m: string) => void }} opts
|
|
54
56
|
* journeys: config journey ids or ready-made journeys (purchase, add_to_cart, sign_up, lead, search).
|
|
55
57
|
* goals: extra journeys described in plain language. events: only check these events.
|
|
56
58
|
* providers: tracking providers to check (default: config "providers", else GA4, Google Ads, Meta, TikTok).
|
|
57
59
|
* saveSite: record the site's responses and the steps taken into this folder, for replaySite (see replay.js).
|
|
58
60
|
* replaySite: answer every request from a folder saved with saveSite, without the network.
|
|
61
|
+
* against: { id, file } of the plan given with --against; opts.plan is then that plan. Used alone, the
|
|
62
|
+
* journeys come from the plan; with journeys or goals, only the plan events expected on them are checked.
|
|
59
63
|
*/
|
|
60
64
|
export async function runAudit(opts) {
|
|
61
65
|
const { gtmContainer } = opts;
|
|
@@ -68,19 +72,29 @@ export async function runAudit(opts) {
|
|
|
68
72
|
const useAi = opts.ai !== false;
|
|
69
73
|
if (!useAi && !opts.scripted) throw new Error("Without an AI model, journeys must be replayed from their fixed steps (scripted).");
|
|
70
74
|
|
|
71
|
-
const
|
|
75
|
+
const against = opts.against ? { ...opts.against, label: opts.against.id ?? path.relative(process.cwd(), opts.against.file) } : null;
|
|
76
|
+
const { journeys, plan } = planRun({ config, plan: opts.plan, picks: opts.journeys, goals: opts.goals, events, against: Boolean(against) });
|
|
72
77
|
config = { ...config, providers: resolveProviders(opts.providers ?? config.providers, plan) };
|
|
73
78
|
const viewports = opts.viewports?.length ? opts.viewports : config.viewports ?? ["desktop"];
|
|
74
79
|
const allowed = Object.keys(VIEWPORTS).join(", ");
|
|
75
80
|
if (!Array.isArray(viewports)) throw new Error(`"viewports" must be a list, e.g. ["desktop", "mobile"]. Allowed values: ${allowed}.`);
|
|
76
81
|
const unknown = viewports.filter((v) => !Object.hasOwn(VIEWPORTS, v));
|
|
77
82
|
if (unknown.length) throw new Error(`Unknown viewport ${unknown.map((v) => `"${v}"`).join(", ")}. Use: ${allowed}.`);
|
|
78
|
-
if (!journeys.length) throw new Error(
|
|
83
|
+
if (!journeys.length) throw new Error(`No journeys to run. A run needs one of:\n --journey <name> one of: ${Object.keys(JOURNEY_PRESETS).join(", ")} (or a journey id from your config)\n --goal "<text>" describe what to do in your own words\n --config <file> a config file with "journeys"\nNot sure what to test? "aitd plan --site ${config.site}" drafts a measurement plan.`);
|
|
79
84
|
if (opts.scripted) {
|
|
80
85
|
const unscripted = journeys.filter((j) => !j.steps?.length && !j.incomplete).map((j) => j.id);
|
|
81
86
|
if (unscripted.length) throw new Error(`Journeys without fixed steps can't be replayed with --scripted: ${unscripted.join(", ")}.`);
|
|
82
87
|
}
|
|
83
88
|
|
|
89
|
+
if (against) {
|
|
90
|
+
const notChecked = plan.events.filter((e) => Array.isArray(e.journeys) && !e.journeys.some((j) => journeys.some((jj) => jj.id === j))).length;
|
|
91
|
+
log(`Checking against plan ${against.label}: ${journeys.length} ${journeys.length === 1 ? "journey" : "journeys"}${notChecked ? `, ${notChecked} planned ${notChecked === 1 ? "event" : "events"} not checked in this run` : ""}`);
|
|
92
|
+
for (const j of journeys) {
|
|
93
|
+
const expected = plan.events.filter((e) => !e.journeys || e.journeys.includes(j.id)).map((e) => e.name);
|
|
94
|
+
log(` - ${j.id}: ${j.goal ?? "fixed steps"}${expected.length ? ` [${[...new Set(expected)].join(", ")}]` : ""}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
84
98
|
const provider = useAi ? await createProvider({ name: opts.provider ?? config.provider, model: opts.model ?? config.model, baseUrl: opts.baseUrl ?? config.base_url }) : null;
|
|
85
99
|
if (provider) log(`Using ${provider.name} (${provider.model})`);
|
|
86
100
|
|
|
@@ -116,8 +130,8 @@ export async function runAudit(opts) {
|
|
|
116
130
|
explained = await explainFindings({ findings, plan, runs, gtmContainer, provider, lang, log });
|
|
117
131
|
}
|
|
118
132
|
|
|
119
|
-
const outDir = path.resolve(opts.outDir ??
|
|
120
|
-
const reportPath = await writeReports({ outDir, config, plan, runs, findings, explained, lang, branding: opts.branding ?? config.branding !== false });
|
|
133
|
+
const outDir = path.resolve(opts.outDir ?? (await newRunDir(opts.dataDir, config.site)).dir);
|
|
134
|
+
const reportPath = await writeReports({ outDir, config, plan, runs, findings, explained, lang, against, branding: opts.branding ?? config.branding !== false });
|
|
121
135
|
const counts = summarize(groupFindings(findings));
|
|
122
|
-
return { runs, findings, explained, counts, reportPath, outDir, lang, ...(saveError ? { saveError } : {}) };
|
|
136
|
+
return { journeys, plan, runs, findings, explained, counts, reportPath, outDir, lang, ...(against ? { againstId: against.id } : {}), ...(saveError ? { saveError } : {}) };
|
|
123
137
|
}
|
package/src/checks.js
CHANGED
|
@@ -68,6 +68,17 @@ export function hasParam(hit, name) {
|
|
|
68
68
|
return [name, ...(PARAM_ALIASES[name] ?? [])].some((n) => hit.params?.[n] !== undefined && hit.params[n] !== "");
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/** A hit's parameter under its own name or the first common alias it carries. */
|
|
72
|
+
function paramOf(hit, name) {
|
|
73
|
+
for (const n of [name, ...(PARAM_ALIASES[name] ?? [])]) {
|
|
74
|
+
if (hit.params?.[n] !== undefined && hit.params[n] !== "") return hit.params[n];
|
|
75
|
+
}
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const isPurchase = (hit) => canonicalEvent(hit.name) === "purchase";
|
|
80
|
+
const EMPTY_LIST = /^\s*(\[\s*\])?\s*$/;
|
|
81
|
+
|
|
71
82
|
/** The page load a hit came from: GA4's page load id (_p) when it has one, else the page address. */
|
|
72
83
|
function pageOf(hit) {
|
|
73
84
|
try {
|
|
@@ -99,7 +110,7 @@ function expectedIn(planEvent, journeyId) {
|
|
|
99
110
|
return !planEvent.journeys || planEvent.journeys.includes(journeyId);
|
|
100
111
|
}
|
|
101
112
|
|
|
102
|
-
function isConsentCommand(entry, kind) {
|
|
113
|
+
export function isConsentCommand(entry, kind) {
|
|
103
114
|
return Array.isArray(entry.value) && entry.value[0] === "consent" && entry.value[1] === kind;
|
|
104
115
|
}
|
|
105
116
|
|
|
@@ -203,6 +214,62 @@ export function checkRun(plan, run, { lang = "en" } = {}) {
|
|
|
203
214
|
});
|
|
204
215
|
}
|
|
205
216
|
|
|
217
|
+
// Purchase values: a value that is 0, negative or not a number, a currency that isn't an ISO
|
|
218
|
+
// code, and an empty items list (reported once per platform and event).
|
|
219
|
+
const purchaseReported = new Set();
|
|
220
|
+
const reportPurchase = (hit, check, severity, detail) => {
|
|
221
|
+
const k = `${check}|${key(hit.platform, hit.name)}`;
|
|
222
|
+
if (purchaseReported.has(k)) return;
|
|
223
|
+
purchaseReported.add(k);
|
|
224
|
+
findings.push({ ...base, severity, check, platform: hit.platform, event: hit.name, step: stepLabel(hit.step), detail, evidence: JSON.stringify(hit.params) });
|
|
225
|
+
};
|
|
226
|
+
const purchases = run.hits.filter(isPurchase);
|
|
227
|
+
for (const hit of purchases) {
|
|
228
|
+
const value = paramOf(hit, "value");
|
|
229
|
+
if (value !== undefined) {
|
|
230
|
+
const n = Number(value);
|
|
231
|
+
if (!Number.isFinite(n) || String(value).trim() === "") reportPurchase(hit, "purchase_value_invalid", "broken", d.purchaseValueNaN(hit.platform, hit.name, value));
|
|
232
|
+
else if (n <= 0) reportPurchase(hit, "purchase_value_invalid", "broken", d.purchaseValueZero(hit.platform, hit.name, value));
|
|
233
|
+
}
|
|
234
|
+
const currency = paramOf(hit, "currency");
|
|
235
|
+
if (currency !== undefined && !/^[A-Z]{3}$/.test(currency)) reportPurchase(hit, "purchase_currency_invalid", "broken", d.purchaseCurrency(hit.platform, hit.name, currency));
|
|
236
|
+
// GA4 sends items as pr1, pr2...; Meta as contents or content_ids. Items left out of a GA4
|
|
237
|
+
// purchase count as empty, unless the plan requires them and missing_params already says so.
|
|
238
|
+
const listed = ["items", "contents", "content_ids"].filter((n) => hit.params?.[n] !== undefined);
|
|
239
|
+
const requiresItems = planEventFor(hit)?.required_params?.includes("items");
|
|
240
|
+
const empty = listed.length
|
|
241
|
+
? listed.every((n) => EMPTY_LIST.test(hit.params[n]))
|
|
242
|
+
: hit.platform === "ga4" && !requiresItems && !hasParam(hit, "items");
|
|
243
|
+
if (empty) reportPurchase(hit, "purchase_items_empty", "warning", d.purchaseItemsEmpty(hit.platform, hit.name));
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// GA4 and Meta reporting the same order with a different value or currency. Purchases are paired
|
|
247
|
+
// by order ID; without IDs, a run with one purchase on each platform is taken as one order.
|
|
248
|
+
const ga4Buys = purchases.filter((h) => h.platform === "ga4");
|
|
249
|
+
const metaBuys = purchases.filter((h) => h.platform === "meta");
|
|
250
|
+
const orderId = (h) => paramOf(h, "transaction_id");
|
|
251
|
+
const pairs = [];
|
|
252
|
+
for (const g of ga4Buys) {
|
|
253
|
+
const m = orderId(g) !== undefined && metaBuys.find((h) => orderId(h) === orderId(g));
|
|
254
|
+
if (m) pairs.push([g, m]);
|
|
255
|
+
}
|
|
256
|
+
if (!pairs.length && ga4Buys.length === 1 && metaBuys.length === 1) {
|
|
257
|
+
const [g, m] = [ga4Buys[0], metaBuys[0]];
|
|
258
|
+
if (orderId(g) === undefined || orderId(m) === undefined) pairs.push([g, m]);
|
|
259
|
+
}
|
|
260
|
+
const mismatchSeen = new Set();
|
|
261
|
+
for (const [g, m] of pairs) {
|
|
262
|
+
const [gv, mv] = [paramOf(g, "value"), paramOf(m, "value")];
|
|
263
|
+
const [gc, mc] = [paramOf(g, "currency"), paramOf(m, "currency")];
|
|
264
|
+
const diffs = [];
|
|
265
|
+
if (gv !== undefined && mv !== undefined && Number.isFinite(Number(gv)) && Number.isFinite(Number(mv)) && Math.abs(Number(gv) - Number(mv)) > 0.005) diffs.push(["value", gv, mv]);
|
|
266
|
+
if (gc !== undefined && mc !== undefined && gc.toUpperCase() !== mc.toUpperCase()) diffs.push(["currency", gc, mc]);
|
|
267
|
+
const k = `${orderId(g) ?? ""}|${diffs.map((x) => x.join("=")).join(",")}`;
|
|
268
|
+
if (!diffs.length || mismatchSeen.has(k)) continue;
|
|
269
|
+
mismatchSeen.add(k);
|
|
270
|
+
findings.push({ ...base, severity: "warning", check: "purchase_mismatch", platform: "meta", event: m.name, step: stepLabel(m.step), detail: d.purchaseMismatch(orderId(g) ?? orderId(m), diffs), evidence: JSON.stringify({ ga4: g.params, meta: m.params }) });
|
|
271
|
+
}
|
|
272
|
+
|
|
206
273
|
// Server-side tagging. Only what the browser can see: a GA4 hit to a server container, and requests
|
|
207
274
|
// to Stape or Elevar. Meta's and TikTok's server-side copies never pass through the browser.
|
|
208
275
|
const serverHits = run.hits.filter((h) => h.delivery === "server");
|
|
@@ -377,3 +444,58 @@ export function groupFindings(findings) {
|
|
|
377
444
|
}
|
|
378
445
|
return [...groups.values()].sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
|
|
379
446
|
}
|
|
447
|
+
|
|
448
|
+
const WRONG_VALUE_CHECKS = new Set(["purchase_value_invalid", "purchase_currency_invalid", "purchase_items_empty", "purchase_mismatch"]);
|
|
449
|
+
export const PLAN_BUCKETS = ["implemented", "missing", "missing_params", "wrong_values", "unplanned", "not_checked"];
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* Sorts every planned event into one bucket from the runs and the findings checkRun gave them:
|
|
453
|
+
* implemented, missing, missing_params, wrong_values or not_checked, plus one unplanned entry per
|
|
454
|
+
* event that fired but is not in the plan. Coverage = implemented / planned events that were checkable.
|
|
455
|
+
* @param {Plan} plan
|
|
456
|
+
* @param {any[]} runs
|
|
457
|
+
* @param {Finding[]} findings
|
|
458
|
+
*/
|
|
459
|
+
export function planCheck(plan, runs, findings) {
|
|
460
|
+
const events = (plan?.events ?? []).map((e) => ({ ...e, platform: String(e.platform ?? "").toLowerCase() }));
|
|
461
|
+
const entries = [];
|
|
462
|
+
for (const ev of events) {
|
|
463
|
+
const walked = runs.filter((r) => expectedIn(ev, r.journeyId) && (!r.providers || r.providers.includes(ev.platform)));
|
|
464
|
+
const entry = { platform: ev.platform, event: ev.name, bucket: "not_checked", journeys: [...new Set(walked.map((r) => r.journeyId))] };
|
|
465
|
+
if (walked.length) {
|
|
466
|
+
const own = findings.filter((f) => f.event && walked.some((r) => r.journeyId === f.journey && r.viewport === f.viewport) && hitMatchesEvent(ev, { platform: f.platform, name: f.event }));
|
|
467
|
+
const params = own.filter((f) => f.check === "missing_params");
|
|
468
|
+
const values = own.filter((f) => WRONG_VALUE_CHECKS.has(f.check));
|
|
469
|
+
const fired = walked.some((r) => r.hits.some((h) => hitMatchesEvent(ev, h)));
|
|
470
|
+
if (params.length) entry.bucket = "missing_params";
|
|
471
|
+
else if (values.length) entry.bucket = "wrong_values";
|
|
472
|
+
else entry.bucket = fired ? "implemented" : "missing";
|
|
473
|
+
const details = [...new Set([...params, ...values].map((f) => f.detail))];
|
|
474
|
+
if (details.length) entry.details = details;
|
|
475
|
+
}
|
|
476
|
+
entries.push(entry);
|
|
477
|
+
}
|
|
478
|
+
// Unplanned events, from the runs whose unplanned_events finding survived the --event filter.
|
|
479
|
+
const seen = new Set();
|
|
480
|
+
for (const r of runs) {
|
|
481
|
+
if (!findings.some((f) => f.check === "unplanned_events" && f.journey === r.journeyId && f.viewport === r.viewport)) continue;
|
|
482
|
+
for (const h of r.hits) {
|
|
483
|
+
if (events.some((e) => hitMatchesEvent(e, h)) || seen.has(key(h.platform, h.name))) continue;
|
|
484
|
+
seen.add(key(h.platform, h.name));
|
|
485
|
+
entries.push({ platform: h.platform, event: h.name, bucket: "unplanned", journeys: [r.journeyId] });
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
const score = (list) => {
|
|
489
|
+
const planned = list.filter((e) => e.bucket !== "unplanned");
|
|
490
|
+
const checkable = planned.filter((e) => e.bucket !== "not_checked").length;
|
|
491
|
+
const implemented = planned.filter((e) => e.bucket === "implemented").length;
|
|
492
|
+
return { implemented, checkable, not_checked: planned.length - checkable, percent: checkable ? Math.round((implemented / checkable) * 100) : null };
|
|
493
|
+
};
|
|
494
|
+
const platforms = [...new Set(entries.filter((e) => e.bucket !== "unplanned").map((e) => e.platform))];
|
|
495
|
+
const counts = Object.fromEntries(PLAN_BUCKETS.map((b) => [b, entries.filter((e) => e.bucket === b).length]));
|
|
496
|
+
return {
|
|
497
|
+
coverage: { ...score(entries), platforms: Object.fromEntries(platforms.map((p) => [p, score(entries.filter((e) => e.platform === p))])) },
|
|
498
|
+
counts,
|
|
499
|
+
events: entries,
|
|
500
|
+
};
|
|
501
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -3,74 +3,14 @@ import { readFileSync } from "node:fs";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { parseArgs } from "node:util";
|
|
5
5
|
import { loadConfig, runAudit } from "./audit.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
6
|
+
import { messages } from "./i18n.js";
|
|
7
|
+
import { helpText } from "./help.js";
|
|
8
|
+
import { replaySavedSite, writeSavedSteps } from "./replay.js";
|
|
9
|
+
import { savePlan } from "./plan.js";
|
|
10
|
+
import { DEFAULT_DATA_DIR } from "./workspace.js";
|
|
11
11
|
|
|
12
12
|
const VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
13
13
|
|
|
14
|
-
const USAGE = `ai-tag-debugger: an open-source AI agent that QA-tests your analytics and ad tracking.
|
|
15
|
-
|
|
16
|
-
Usage:
|
|
17
|
-
ai-tag-debugger run --site <url> --journey purchase [options] Audit ready-made journeys, no config needed.
|
|
18
|
-
ai-tag-debugger run --site <url> --goal "<what to do>" Audit a journey you describe in plain language.
|
|
19
|
-
ai-tag-debugger run --config <file> [options] Run an audit from a config file.
|
|
20
|
-
ai-tag-debugger replay <dir>... [--update] Audit sites saved with --save-site again, offline and without AI,
|
|
21
|
-
and list findings that changed. Exits with code 1 when any did.
|
|
22
|
-
ai-tag-debugger mcp [--http] [--port <n>] Start the MCP server for Claude, ChatGPT, Gemini and other chat apps.
|
|
23
|
-
|
|
24
|
-
Run options:
|
|
25
|
-
--site <url> The site to audit (overrides "site" in the config).
|
|
26
|
-
--config <file> Audit config (JSON). See examples/demo.config.json.
|
|
27
|
-
--journey <id> Run this journey (repeatable): a journey id from the config, or a ready-made
|
|
28
|
-
journey: ${Object.keys(JOURNEY_PRESETS).join(", ")}.
|
|
29
|
-
--goal "<text>" Run a journey you describe in plain language (repeatable).
|
|
30
|
-
--events <names> Only check these events, comma separated, e.g. purchase,add_to_cart.
|
|
31
|
-
--max-steps <n> Most AI steps per journey (default 30). Lower it to cap API usage.
|
|
32
|
-
--out <dir> Where to write report.html, report.md, timeline.csv and capture.json (default: reports/<timestamp>).
|
|
33
|
-
--provider <name> AI provider: ${Object.keys(PROVIDERS).join(", ")} (default: whichever API key is set).
|
|
34
|
-
--model <id> Model id (defaults: ${Object.entries(PROVIDERS).filter(([, p]) => p.defaultModel).map(([k, p]) => `${k} ${p.defaultModel}`).join(", ")}).
|
|
35
|
-
--base-url <url> API address for --provider openai-compatible (Kimi, GLM, DeepSeek, OpenRouter, Ollama...).
|
|
36
|
-
--lang <code> Report language: ${Object.keys(LANGUAGES).join(" or ")} (default: en, or "language" in the config).
|
|
37
|
-
--providers <list> Tracking providers to check, comma separated (default: ${DEFAULT_PROVIDERS.join(",")}).
|
|
38
|
-
Choose from ${Object.keys(TRACKING_PROVIDERS).join(", ")}. Requests to the
|
|
39
|
-
others are still answered locally, but only counted ("providers" in the config).
|
|
40
|
-
--viewport <name> Only run this viewport: desktop or mobile (repeatable).
|
|
41
|
-
--allow-host <host> Let journeys visit this host too, e.g. a hosted checkout (repeatable). By default the
|
|
42
|
-
browser stays on the audited site and its subdomains ("allowed_hosts" in the config).
|
|
43
|
-
--scripted Replay each journey's fixed "steps" instead of letting the AI browse.
|
|
44
|
-
--no-ai Use no AI model at all (requires --scripted). The report then has no explanations.
|
|
45
|
-
--fail-on <level> Exit with code 1 when findings reach this level: broken, risk, warning or never (default: broken).
|
|
46
|
-
--headed Show the browser window.
|
|
47
|
-
--keep-bodies Store tracking requests as sent. By default, emails, phone numbers and user ids sent
|
|
48
|
-
for ad matching are redacted in the report and capture.json ("keep_bodies" in the config).
|
|
49
|
-
--no-branding Leave the Analitik İşler logo and credit line out of the reports ("branding": false in the config).
|
|
50
|
-
--save-site <dir> Also save the site's responses and the steps taken in this folder, so "replay" can audit
|
|
51
|
-
the same pages again offline. Its findings become the expected ones.
|
|
52
|
-
|
|
53
|
-
Replay options:
|
|
54
|
-
--update Make this replay's findings the expected ones (after checking that the changes are right).
|
|
55
|
-
--out <dir> Where to write the replay's reports (default: <dir>/last-replay).
|
|
56
|
-
|
|
57
|
-
MCP options:
|
|
58
|
-
--http Serve over Streamable HTTP instead of stdio (for apps that connect to a URL).
|
|
59
|
-
--port <n> HTTP port (default 8787).
|
|
60
|
-
--host <addr> HTTP host (default 127.0.0.1).
|
|
61
|
-
--token <secret> Secret URL path segment, at least 16 characters (default: random, printed at start).
|
|
62
|
-
--allow-private-network Over HTTP, let audits open localhost and private network addresses (blocked by default).
|
|
63
|
-
--reports <dir> Where finished audits write reports (default: ~/ai-tag-debugger-reports).
|
|
64
|
-
--public-url <url> The HTTPS address your tunnel or host gives this server, so audits can link to their reports.
|
|
65
|
-
|
|
66
|
-
-h, --help Show this help.
|
|
67
|
-
-v, --version Print the version.
|
|
68
|
-
|
|
69
|
-
Short alias: aitd (aitd run ... is the same as ai-tag-debugger run ...).
|
|
70
|
-
|
|
71
|
-
API keys: set one of ${Object.values(PROVIDERS).map((p) => p.env).join(", ")}.
|
|
72
|
-
Any OpenAI-compatible API: --provider openai-compatible --base-url <url> --model <id>, with the key in OPENAI_COMPATIBLE_API_KEY.`;
|
|
73
|
-
|
|
74
14
|
async function main() {
|
|
75
15
|
const { values, positionals } = parseArgs({
|
|
76
16
|
allowPositionals: true,
|
|
@@ -83,6 +23,7 @@ async function main() {
|
|
|
83
23
|
out: { type: "string" },
|
|
84
24
|
journey: { type: "string", multiple: true },
|
|
85
25
|
viewport: { type: "string", multiple: true },
|
|
26
|
+
platforms: { type: "string", multiple: true },
|
|
86
27
|
providers: { type: "string", multiple: true },
|
|
87
28
|
"allow-host": { type: "string", multiple: true },
|
|
88
29
|
"allow-private-network": { type: "boolean", default: false },
|
|
@@ -103,7 +44,13 @@ async function main() {
|
|
|
103
44
|
reports: { type: "string" },
|
|
104
45
|
"public-url": { type: "string" },
|
|
105
46
|
"save-site": { type: "string" },
|
|
47
|
+
"save-steps": { type: "string" },
|
|
106
48
|
update: { type: "boolean", default: false },
|
|
49
|
+
"data-dir": { type: "string" },
|
|
50
|
+
against: { type: "string" },
|
|
51
|
+
from: { type: "string" },
|
|
52
|
+
format: { type: "string" },
|
|
53
|
+
force: { type: "boolean", default: false },
|
|
107
54
|
help: { type: "boolean", short: "h", default: false },
|
|
108
55
|
version: { type: "boolean", short: "v", default: false },
|
|
109
56
|
},
|
|
@@ -141,19 +88,75 @@ async function main() {
|
|
|
141
88
|
if (changed && !values.update) process.exitCode = 1;
|
|
142
89
|
return;
|
|
143
90
|
}
|
|
144
|
-
if (
|
|
145
|
-
|
|
146
|
-
|
|
91
|
+
if (command === "plan" && positionals[1] === "list" && !values.help) {
|
|
92
|
+
if (!values.site) throw new Error("plan list needs --site <url>: aitd plan list --site https://www.example.com");
|
|
93
|
+
const { domainOf, listPlanIds } = await import("./workspace.js");
|
|
94
|
+
const ids = await listPlanIds(values["data-dir"] ?? DEFAULT_DATA_DIR, values.site);
|
|
95
|
+
if (!ids.length) throw new Error(`No plan yet for ${domainOf(values.site)}. Draft one with: aitd plan --site ${values.site}`);
|
|
96
|
+
for (const id of ids) console.log(id);
|
|
97
|
+
return;
|
|
147
98
|
}
|
|
148
|
-
if (
|
|
149
|
-
|
|
99
|
+
if (command === "plan" && !values.help && ["import", "export"].includes(positionals[1])) {
|
|
100
|
+
await planSheet(positionals[1], positionals[2], values);
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (command === "plan" && !values.help) {
|
|
104
|
+
if (values.from) throw new Error("--from was removed. Draft a plan with: aitd plan --site <url>");
|
|
105
|
+
if (values.out) throw new Error("plan --out was removed. Plans are saved in aitd/<domain>/plans/<id>/; use --data-dir <dir> to keep them elsewhere.");
|
|
106
|
+
if (values.force) throw new Error("--force was removed. Each plan is saved in a new folder, so nothing is replaced.");
|
|
107
|
+
if (!values.site) throw new Error("plan needs --site <url>: aitd plan --site https://www.example.com");
|
|
108
|
+
const dataDir = values["data-dir"] ?? DEFAULT_DATA_DIR;
|
|
109
|
+
const { domainOf } = await import("./workspace.js");
|
|
110
|
+
domainOf(values.site); // Checked before the browser starts.
|
|
111
|
+
const maxSteps = values["max-steps"] ? Number(values["max-steps"]) : undefined;
|
|
112
|
+
if (maxSteps !== undefined && !(Number.isInteger(maxSteps) && maxSteps > 0)) throw new Error("--max-steps must be a positive whole number.");
|
|
113
|
+
const { createProvider } = await import("./providers.js");
|
|
114
|
+
const { discoverPlan } = await import("./discover.js");
|
|
115
|
+
const provider = await createProvider({ name: values.provider, model: values.model, baseUrl: values["base-url"] });
|
|
116
|
+
console.log(`Exploring ${values.site} to draft its measurement plan…`);
|
|
117
|
+
const { plan } = await discoverPlan({ site: values.site, provider, maxSteps, headed: values.headed, allowedHosts: values["allow-host"], log: console.log });
|
|
118
|
+
const { id, dir } = await savePlan(plan, { dataDir, site: values.site, branding: !values["no-branding"] });
|
|
119
|
+
const custom = plan.events.filter((e) => e.custom).length;
|
|
120
|
+
const rel = path.relative(process.cwd(), dir);
|
|
121
|
+
console.log(`✓ Saved plan ${id}: ${plan.events.length} events (${custom} custom) and ${plan.user_properties.length} user properties in ${rel.startsWith("..") ? dir : rel}`);
|
|
122
|
+
return;
|
|
150
123
|
}
|
|
124
|
+
if (values.help || command !== "run" || (!values.config && !values.site && !values.against)) {
|
|
125
|
+
console.log(helpText(command));
|
|
126
|
+
process.exit(values.help ? 0 : 2);
|
|
127
|
+
}
|
|
128
|
+
if (values["no-ai"]) values.scripted = true; // Without an AI model the journeys can only replay fixed steps.
|
|
129
|
+
if (values.providers?.length) console.log("--providers is now --platforms.");
|
|
130
|
+
const platforms = [...(values.platforms ?? []), ...(values.providers ?? [])];
|
|
151
131
|
const levels = ["broken", "risk", "warning"];
|
|
152
132
|
const failOn = values["fail-on"];
|
|
153
133
|
if (failOn !== "never" && !levels.includes(failOn)) throw new Error(`--fail-on must be one of ${levels.join(", ")}, never.`);
|
|
154
134
|
|
|
155
135
|
const loaded = values.config ? await loadConfig(values.config) : { config: {}, plan: null, gtmContainer: null };
|
|
156
136
|
if (values.site) loaded.config = { ...loaded.config, site: values.site };
|
|
137
|
+
if (!values.against && !values.journey?.length && !values.goal?.length && !loaded.config.journeys?.length) {
|
|
138
|
+
throw new Error('Nothing to test. Add --against latest, --journey <id> or --goal "<text>".');
|
|
139
|
+
}
|
|
140
|
+
let against = null;
|
|
141
|
+
if (values.against) {
|
|
142
|
+
const { readJson } = await import("./audit.js");
|
|
143
|
+
const { resolvePlan } = await import("./workspace.js");
|
|
144
|
+
if (loaded.config.site) {
|
|
145
|
+
try {
|
|
146
|
+
new URL(loaded.config.site);
|
|
147
|
+
} catch {
|
|
148
|
+
throw new Error(`"${loaded.config.site}" is not a full URL. Use e.g. https://www.example.com`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
against = await resolvePlan(values.against, { dataDir: values["data-dir"] ?? DEFAULT_DATA_DIR, site: loaded.config.site });
|
|
152
|
+
const plan = await readJson(against.file);
|
|
153
|
+
if (!Array.isArray(plan?.events)) throw new Error(`${against.file} is not a plan: it needs an "events" list.`);
|
|
154
|
+
if (!loaded.config.site) {
|
|
155
|
+
if (!plan.site) throw new Error(`${against.file} has no "site". Add --site <url>.`);
|
|
156
|
+
loaded.config = { ...loaded.config, site: plan.site };
|
|
157
|
+
}
|
|
158
|
+
loaded.plan = plan;
|
|
159
|
+
}
|
|
157
160
|
if (values["keep-bodies"]) loaded.config = { ...loaded.config, keep_bodies: true };
|
|
158
161
|
if (values["allow-host"]?.length) loaded.config = { ...loaded.config, allowed_hosts: [...(loaded.config.allowed_hosts ?? []), ...values["allow-host"]] };
|
|
159
162
|
try {
|
|
@@ -163,6 +166,8 @@ async function main() {
|
|
|
163
166
|
}
|
|
164
167
|
const maxSteps = values["max-steps"] ? Number(values["max-steps"]) : undefined;
|
|
165
168
|
if (maxSteps !== undefined && !(Number.isInteger(maxSteps) && maxSteps > 0)) throw new Error("--max-steps must be a positive whole number.");
|
|
169
|
+
// run --out is a hidden alias kept for one minor version; --data-dir replaces it.
|
|
170
|
+
if (values.out) console.log("Note: run --out will be removed in a later version. Runs are saved in aitd/<domain>/runs/<id>/; use --data-dir <dir> to keep them elsewhere.");
|
|
166
171
|
const result = await runAudit({
|
|
167
172
|
...loaded,
|
|
168
173
|
journeys: values.journey,
|
|
@@ -170,7 +175,7 @@ async function main() {
|
|
|
170
175
|
events: (values.events ?? []).flatMap((e) => e.split(",")).map((e) => e.trim()).filter(Boolean),
|
|
171
176
|
maxSteps,
|
|
172
177
|
viewports: values.viewport,
|
|
173
|
-
providers:
|
|
178
|
+
providers: platforms.length ? platforms.flatMap((p) => p.split(",")) : undefined,
|
|
174
179
|
scripted: values.scripted,
|
|
175
180
|
ai: !values["no-ai"],
|
|
176
181
|
provider: values.provider,
|
|
@@ -180,7 +185,9 @@ async function main() {
|
|
|
180
185
|
headed: values.headed,
|
|
181
186
|
branding: values["no-branding"] ? false : undefined,
|
|
182
187
|
outDir: values.out,
|
|
188
|
+
dataDir: values["data-dir"],
|
|
183
189
|
saveSite: values["save-site"],
|
|
190
|
+
against,
|
|
184
191
|
log: (msg) => console.log(msg),
|
|
185
192
|
});
|
|
186
193
|
|
|
@@ -193,6 +200,44 @@ async function main() {
|
|
|
193
200
|
if (levels.slice(0, threshold + 1).some((l) => result.counts[l] > 0)) process.exitCode = 1;
|
|
194
201
|
}
|
|
195
202
|
if (result.saveError) process.exitCode = 1;
|
|
203
|
+
if (values["save-steps"]) {
|
|
204
|
+
try {
|
|
205
|
+
const { saved, skipped } = await writeSavedSteps(values["save-steps"], { site: loaded.config.site, plan: result.plan, journeys: result.journeys, runs: result.runs });
|
|
206
|
+
if (saved.length) console.log(`Saved the steps of ${saved.join(", ")} in ${values["save-steps"]}. Repeat them with: aitd run --config ${values["save-steps"]} --scripted --no-ai`);
|
|
207
|
+
if (skipped.length) console.log(`No steps to save for ${skipped.join(", ")}.`);
|
|
208
|
+
} catch (err) {
|
|
209
|
+
console.log(`Could not save the steps in ${values["save-steps"]}: ${err.message}`);
|
|
210
|
+
process.exitCode = 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function planSheet(action, arg, values) {
|
|
216
|
+
const dataDir = values["data-dir"] ?? DEFAULT_DATA_DIR;
|
|
217
|
+
const shown = (p) => {
|
|
218
|
+
const rel = path.relative(process.cwd(), p);
|
|
219
|
+
return rel.startsWith("..") ? p : rel;
|
|
220
|
+
};
|
|
221
|
+
const { readPlanSheet, writePlanSheet } = await import("./sheet.js");
|
|
222
|
+
if (action === "import") {
|
|
223
|
+
if (!arg) throw new Error("plan import needs a file: aitd plan import plan.xlsx --site https://www.example.com");
|
|
224
|
+
if (!values.site) throw new Error(`plan import needs --site <url>: aitd plan import ${arg} --site https://www.example.com`);
|
|
225
|
+
const { domainOf } = await import("./workspace.js");
|
|
226
|
+
domainOf(values.site);
|
|
227
|
+
const { plan, warnings } = await readPlanSheet(arg, { site: values.site });
|
|
228
|
+
for (const w of warnings) console.log(`Warning: ${w}`);
|
|
229
|
+
const { id, dir } = await savePlan(plan, { dataDir, site: values.site });
|
|
230
|
+
console.log(`✓ Imported plan ${id}: ${plan.events.length} events and ${plan.user_properties.length} user properties in ${shown(dir)}`);
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
if (!arg) throw new Error("plan export needs a plan: aitd plan export latest --site https://www.example.com");
|
|
234
|
+
const format = (values.format ?? "xlsx").toLowerCase();
|
|
235
|
+
if (!["xlsx", "csv"].includes(format)) throw new Error(`--format must be xlsx or csv (found "${values.format}").`);
|
|
236
|
+
const { resolvePlan } = await import("./workspace.js");
|
|
237
|
+
const { file } = await resolvePlan(arg, { dataDir, site: values.site });
|
|
238
|
+
const plan = JSON.parse(readFileSync(file, "utf8"));
|
|
239
|
+
const written = await writePlanSheet(plan, path.dirname(file), { format });
|
|
240
|
+
console.log(`✓ Exported the plan to ${written.map(shown).join(" and ")}. Edit it, then save it as a new plan with: aitd plan import ${shown(written[0])} --site ${values.site ?? plan.site ?? "<url>"}`);
|
|
196
241
|
}
|
|
197
242
|
|
|
198
243
|
main().catch((err) => {
|